From 69f0cfa04382858ee00174564481bc26e8537b48 Mon Sep 17 00:00:00 2001 From: Adnan Date: Tue, 23 Dec 2025 18:49:17 +0100 Subject: [PATCH 01/17] PCSM-219: Add Viper dependency and create config/config.go - Add github.com/spf13/viper to go.mod - Create config/config.go with Init() function - Bind env vars for global options only (not clone tuning) PCSM-219: Integrate Viper with Cobra for logging and server options - Call config.Init() in PersistentPreRun - Use Viper for log-level, log-json, no-color flags - Use Viper for source/target URIs and port - Simplify getPort() to use Viper with env var fallback --- go.mod | 15 ++++++++-- go.sum | 30 +++++++++++++++++--- main.go | 88 +++++++++++++++++++-------------------------------------- 3 files changed, 67 insertions(+), 66 deletions(-) diff --git a/go.mod b/go.mod index 9e781f0..05456ce 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,9 @@ require ( github.com/prometheus/client_golang v1.22.0 github.com/rs/zerolog v1.34.0 github.com/spf13/cobra v1.9.1 - github.com/spf13/pflag v1.0.6 - github.com/stretchr/testify v1.10.0 + github.com/spf13/pflag v1.0.10 + github.com/spf13/viper v1.21.0 + github.com/stretchr/testify v1.11.1 go.mongodb.org/mongo-driver/v2 v2.2.1 golang.org/x/sync v0.18.0 ) @@ -17,21 +18,29 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/compress v1.18.0 // indirect - github.com/kr/text v0.2.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.45.0 // indirect golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.31.0 // indirect diff --git a/go.sum b/go.sum index 42f8a11..30eb651 100644 --- a/go.sum +++ b/go.sum @@ -4,11 +4,16 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -31,6 +36,8 @@ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APP github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -48,12 +55,25 @@ github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -65,6 +85,8 @@ github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfS github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mongodb.org/mongo-driver/v2 v2.2.1 h1:w5xra3yyu/sGrziMzK1D0cRRaH/b7lWCSsoN6+WV6AM= go.mongodb.org/mongo-driver/v2 v2.2.1/go.mod h1:qQkDMhCGWl3FN509DfdPd4GRBLU/41zqF/k8eTRceps= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= diff --git a/main.go b/main.go index e969466..737ed69 100644 --- a/main.go +++ b/main.go @@ -10,7 +10,6 @@ import ( "os" "os/signal" "runtime" - "strconv" "strings" "time" @@ -20,6 +19,7 @@ import ( "github.com/rs/zerolog" "github.com/spf13/cobra" "github.com/spf13/pflag" + "github.com/spf13/viper" "go.mongodb.org/mongo-driver/v2/mongo" "go.mongodb.org/mongo-driver/v2/x/mongo/driver/connstring" @@ -61,9 +61,13 @@ var rootCmd = &cobra.Command{ SilenceUsage: true, PersistentPreRun: func(cmd *cobra.Command, _ []string) { - logLevelFlag, _ := cmd.PersistentFlags().GetString("log-level") - logJSON, _ := cmd.PersistentFlags().GetBool("log-json") - logNoColor, _ := cmd.PersistentFlags().GetBool("no-color") + // Initialize Viper config binding + config.Init(cmd) + + // Get logging configuration from Viper (supports CLI flags and env vars) + logLevelFlag := viper.GetString("log-level") + logJSON := viper.GetBool("log-json") + logNoColor := viper.GetBool("no-color") logLevel, err := zerolog.ParseLevel(logLevelFlag) if err != nil { @@ -81,23 +85,15 @@ var rootCmd = &cobra.Command{ return nil } - port, err := getPort(cmd.Flags()) - if err != nil { - return err - } + port := getPort(cmd.Flags()) - sourceURI, _ := cmd.Flags().GetString("source") - if sourceURI == "" { - sourceURI = os.Getenv("PCSM_SOURCE_URI") - } + // Use Viper to get source/target URIs (supports CLI flags and env vars) + sourceURI := viper.GetString("source") if sourceURI == "" { return errors.New("required flag --source not set") } - targetURI, _ := cmd.Flags().GetString("target") - if targetURI == "" { - targetURI = os.Getenv("PCSM_TARGET_URI") - } + targetURI := viper.GetString("target") if targetURI == "" { return errors.New("required flag --target not set") } @@ -150,10 +146,7 @@ var statusCmd = &cobra.Command{ Use: "status", Short: "Get the status of the replication process", RunE: func(cmd *cobra.Command, _ []string) error { - port, err := getPort(cmd.Flags()) - if err != nil { - return err - } + port := getPort(cmd.Flags()) return NewClient(port).Status(cmd.Context()) }, @@ -164,10 +157,7 @@ var startCmd = &cobra.Command{ Use: "start", Short: "Start Cluster Replication", RunE: func(cmd *cobra.Command, _ []string) error { - port, err := getPort(cmd.Flags()) - if err != nil { - return err - } + port := getPort(cmd.Flags()) pauseOnInitialSync, _ := cmd.Flags().GetBool("pause-on-initial-sync") includeNamespaces, _ := cmd.Flags().GetStringSlice("include-namespaces") @@ -188,10 +178,7 @@ var finalizeCmd = &cobra.Command{ Use: "finalize", Short: "Finalize Cluster Replication", RunE: func(cmd *cobra.Command, _ []string) error { - port, err := getPort(cmd.Flags()) - if err != nil { - return err - } + port := getPort(cmd.Flags()) ignoreHistoryLost, _ := cmd.Flags().GetBool("ignore-history-lost") @@ -208,10 +195,7 @@ var pauseCmd = &cobra.Command{ Use: "pause", Short: "Pause Cluster Replication", RunE: func(cmd *cobra.Command, _ []string) error { - port, err := getPort(cmd.Flags()) - if err != nil { - return err - } + port := getPort(cmd.Flags()) return NewClient(port).Pause(cmd.Context()) }, @@ -222,10 +206,7 @@ var resumeCmd = &cobra.Command{ Use: "resume", Short: "Resume Cluster Replication", RunE: func(cmd *cobra.Command, _ []string) error { - port, err := getPort(cmd.Flags()) - if err != nil { - return err - } + port := getPort(cmd.Flags()) fromFailure, _ := cmd.Flags().GetBool("from-failure") @@ -242,10 +223,7 @@ var resetCmd = &cobra.Command{ Use: "reset", Short: "Reset PCSM state (heartbeat and recovery data)", RunE: func(cmd *cobra.Command, _ []string) error { - targetURI, _ := cmd.Flags().GetString("target") - if targetURI == "" { - targetURI = os.Getenv("PCSM_TARGET_URI") - } + targetURI := viper.GetString("target") if targetURI == "" { return errors.New("required flag --target not set") } @@ -267,10 +245,7 @@ var resetRecoveryCmd = &cobra.Command{ Hidden: true, Short: "Reset recovery state", RunE: func(cmd *cobra.Command, _ []string) error { - targetURI, _ := cmd.InheritedFlags().GetString("target") - if targetURI == "" { - targetURI = os.Getenv("PCSM_TARGET_URI") - } + targetURI := viper.GetString("target") if targetURI == "" { return errors.New("required flag --target not set") } @@ -306,10 +281,7 @@ var resetHeartbeatCmd = &cobra.Command{ Hidden: true, Short: "Reset heartbeat state", RunE: func(cmd *cobra.Command, _ []string) error { - targetURI, _ := cmd.InheritedFlags().GetString("target") - if targetURI == "" { - targetURI = os.Getenv("PCSM_TARGET_URI") - } + targetURI := viper.GetString("target") if targetURI == "" { return errors.New("required flag --target not set") } @@ -339,23 +311,21 @@ var resetHeartbeatCmd = &cobra.Command{ }, } -func getPort(flags *pflag.FlagSet) (int, error) { - port, _ := flags.GetInt("port") +func getPort(flags *pflag.FlagSet) int { + // First check if the flag was explicitly set on command line if flags.Changed("port") { - return port, nil - } + port, _ := flags.GetInt("port") - portVar := os.Getenv("PCSM_PORT") - if portVar == "" { - return port, nil + return port } - parsedPort, err := strconv.ParseInt(portVar, 10, 32) - if err != nil { - return 0, errors.Errorf("invalid environment variable PCSM_PORT='%s'", portVar) + // Use Viper which handles env vars automatically (PCSM_PORT) + port := viper.GetInt("port") + if port == 0 { + port = DefaultServerPort } - return int(parsedPort), nil + return port } func main() { From ffb345ed35f338a19f1128cc503ba4dfcc8bf289 Mon Sep 17 00:00:00 2001 From: Adnan Date: Tue, 23 Dec 2025 18:52:06 +0100 Subject: [PATCH 02/17] PCSM-219: Add CLI flag for MongoDB operation timeout (visible) - Add --mongodb-cli-operation-timeout persistent flag (NOT hidden) - Update MongoDBOperationTimeout() to use Viper - Add deprecated alias OperationMongoDBCliTimeout() for backward compat --- config/values.go | 25 ++++++++++++++++++------- main.go | 5 +++++ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/config/values.go b/config/values.go index 46d4280..c8e8d07 100644 --- a/config/values.go +++ b/config/values.go @@ -9,6 +9,7 @@ import ( "time" "github.com/dustin/go-humanize" + "github.com/spf13/viper" ) // UseCollectionBulkWrite determines whether to use the Collection Bulk Write API @@ -84,13 +85,16 @@ func UseTargetClientCompressors() []string { return rv } -// OperationMongoDBCliTimeout returns the effective timeout for MongoDB client operations. -// If the environment variable `PCSM_MONGODB_CLI_OPERATION_TIMEOUT` is set, it must be a valid -// time duration string (e.g., "30s", "2m", "1h"). Otherwise, the -// DefaultMongoDBCliOperationTimeout is used. -func OperationMongoDBCliTimeout() time.Duration { - if v := strings.TrimSpace(os.Getenv("PCSM_MONGODB_CLI_OPERATION_TIMEOUT")); v != "" { - d, err := time.ParseDuration(v) +// MongoDBOperationTimeout returns the timeout for MongoDB client operations. +// +// Configuration sources (in order of precedence): +// - CLI flag: --mongodb-cli-operation-timeout +// - Env var: PCSM_MONGODB_CLI_OPERATION_TIMEOUT +// - Default: 5m +func MongoDBOperationTimeout() time.Duration { + timeoutStr := viper.GetString("mongodb-cli-operation-timeout") + if timeoutStr != "" { + d, err := time.ParseDuration(timeoutStr) if err == nil && d > 0 { return d } @@ -98,3 +102,10 @@ func OperationMongoDBCliTimeout() time.Duration { return DefaultMongoDBCliOperationTimeout } + +// OperationMongoDBCliTimeout is an alias for MongoDBOperationTimeout for backward compatibility. +// +// Deprecated: Use MongoDBOperationTimeout instead. +func OperationMongoDBCliTimeout() time.Duration { + return MongoDBOperationTimeout() +} diff --git a/main.go b/main.go index 737ed69..5e015ba 100644 --- a/main.go +++ b/main.go @@ -343,6 +343,11 @@ func main() { rootCmd.Flags().MarkHidden("reset-state") //nolint:errcheck rootCmd.Flags().MarkHidden("pause-on-initial-sync") //nolint:errcheck + // MongoDB client options (visible per stakeholder decision #3) + rootCmd.PersistentFlags().String("mongodb-cli-operation-timeout", "", + "Timeout for MongoDB operations (e.g., 30s, 5m)") + // NOTE: NOT marking as hidden per stakeholder decision + statusCmd.Flags().Int("port", DefaultServerPort, "Port number") startCmd.Flags().Int("port", DefaultServerPort, "Port number") From 3f2ae6316d2f7e92fd09002128882ab1799db1bf Mon Sep 17 00:00:00 2001 From: Adnan Date: Tue, 23 Dec 2025 18:52:49 +0100 Subject: [PATCH 03/17] PCSM-219: Add hidden CLI flag for bulk write option - Add --use-collection-bulk-write hidden persistent flag - Update UseCollectionBulkWrite() to use Viper - Env var PCSM_USE_COLLECTION_BULK_WRITE still supported --- config/values.go | 12 ++++++++---- main.go | 5 +++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/config/values.go b/config/values.go index c8e8d07..4120133 100644 --- a/config/values.go +++ b/config/values.go @@ -12,11 +12,15 @@ import ( "github.com/spf13/viper" ) -// UseCollectionBulkWrite determines whether to use the Collection Bulk Write API -// instead of the Client Bulk Write API (introduced in MongoDB v8.0). -// Enabled when the PCSM_USE_COLLECTION_BULK_WRITE environment variable is set to "1". +// UseCollectionBulkWrite returns whether to use collection-level bulk write. +// This is an internal option, not exposed via HTTP API. +// +// Configuration sources (in order of precedence): +// - CLI flag: --use-collection-bulk-write (hidden) +// - Env var: PCSM_USE_COLLECTION_BULK_WRITE +// - Default: false func UseCollectionBulkWrite() bool { - return os.Getenv("PCSM_USE_COLLECTION_BULK_WRITE") == "1" + return viper.GetBool("use-collection-bulk-write") } // CloneNumParallelCollections returns the number of collections cloned in parallel diff --git a/main.go b/main.go index 5e015ba..1daf03e 100644 --- a/main.go +++ b/main.go @@ -348,6 +348,11 @@ func main() { "Timeout for MongoDB operations (e.g., 30s, 5m)") // NOTE: NOT marking as hidden per stakeholder decision + // Internal option (hidden, has env var support per decision #5) + rootCmd.PersistentFlags().Bool("use-collection-bulk-write", false, + "Use collection-level bulk write instead of client bulk write") + rootCmd.PersistentFlags().MarkHidden("use-collection-bulk-write") //nolint:errcheck + statusCmd.Flags().Int("port", DefaultServerPort, "Port number") startCmd.Flags().Int("port", DefaultServerPort, "Port number") From 94a83ca4c686b51404abb2559a746ce141d53815 Mon Sep 17 00:00:00 2001 From: Adnan Date: Tue, 23 Dec 2025 18:53:50 +0100 Subject: [PATCH 04/17] PCSM-219: Add hidden CLI flags for clone tuning options (no env var) - Add --clone-num-parallel-collections hidden flag - Add --clone-num-read-workers hidden flag - Add --clone-num-insert-workers hidden flag - Add --clone-segment-size hidden flag - Add --clone-read-batch-size hidden flag - All clone tuning options use Viper (CLI only, no env var support) --- config/values.go | 68 ++++++++++++++++++++++++++++++++---------------- main.go | 19 ++++++++++++++ 2 files changed, 64 insertions(+), 23 deletions(-) diff --git a/config/values.go b/config/values.go index 4120133..ae86ac6 100644 --- a/config/values.go +++ b/config/values.go @@ -4,7 +4,6 @@ import ( "math" "os" "slices" - "strconv" "strings" "time" @@ -23,36 +22,49 @@ func UseCollectionBulkWrite() bool { return viper.GetBool("use-collection-bulk-write") } -// CloneNumParallelCollections returns the number of collections cloned in parallel -// during the clone process. Default is 0. +// CloneNumParallelCollections returns the number of collections to clone in parallel. +// Configurable via CLI flag only (no env var support per decision #2). +// +// Configuration sources (in order of precedence): +// - CLI flag: --clone-num-parallel-collections +// - Default: 0 (auto) func CloneNumParallelCollections() int { - numColl, _ := strconv.ParseInt(os.Getenv("PCSM_CLONE_NUM_PARALLEL_COLLECTIONS"), 10, 32) - - return int(numColl) + return viper.GetInt("clone-num-parallel-collections") } -// CloneNumReadWorkers returns the number of read workers used during the clone. Default is 0. -// Note: Workers are shared across all collections. +// CloneNumReadWorkers returns the number of read workers. +// Configurable via CLI flag only (no env var support per decision #2). +// +// Configuration sources (in order of precedence): +// - CLI flag: --clone-num-read-workers +// - Default: 0 (auto) func CloneNumReadWorkers() int { - numReadWorker, _ := strconv.ParseInt(os.Getenv("PCSM_CLONE_NUM_READ_WORKERS"), 10, 32) - - return int(numReadWorker) + return viper.GetInt("clone-num-read-workers") } -// CloneNumInsertWorkers returns the number of insert workers used during the clone. Default is 0. -// Note: Workers are shared across all collections. +// CloneNumInsertWorkers returns the number of insert workers. +// Configurable via CLI flag only (no env var support per decision #2). +// +// Configuration sources (in order of precedence): +// - CLI flag: --clone-num-insert-workers +// - Default: 0 (auto) func CloneNumInsertWorkers() int { - numInsertWorker, _ := strconv.ParseInt(os.Getenv("PCSM_CLONE_NUM_INSERT_WORKERS"), 10, 32) - - return int(numInsertWorker) + return viper.GetInt("clone-num-insert-workers") } -// CloneSegmentSizeBytes returns the segment size in bytes used during the clone. -// A segment is a range within a collection (by _id) that enables concurrent read/insert -// operations by splitting the collection into multiple parallelizable units. -// Zero or less enables auto size (per each collection). Default is [AutoCloneSegmentSize]. +// CloneSegmentSizeBytes returns the segment size in bytes. +// Configurable via CLI flag only (no env var support per decision #2). +// +// Configuration sources (in order of precedence): +// - CLI flag: --clone-segment-size +// - Default: AutoCloneSegmentSize (0 = auto) func CloneSegmentSizeBytes() int64 { - segmentSizeBytes, _ := humanize.ParseBytes(os.Getenv("PCSM_CLONE_SEGMENT_SIZE")) + sizeStr := viper.GetString("clone-segment-size") + if sizeStr == "" { + return AutoCloneSegmentSize + } + + segmentSizeBytes, _ := humanize.ParseBytes(sizeStr) if segmentSizeBytes == 0 { return AutoCloneSegmentSize } @@ -60,9 +72,19 @@ func CloneSegmentSizeBytes() int64 { return int64(min(segmentSizeBytes, math.MaxInt64)) //nolint:gosec } -// CloneReadBatchSizeBytes returns the read batch size in bytes used during the clone. Default is 0. +// CloneReadBatchSizeBytes returns the read batch size in bytes. +// Configurable via CLI flag only (no env var support per decision #2). +// +// Configuration sources (in order of precedence): +// - CLI flag: --clone-read-batch-size +// - Default: 0 (uses MaxWriteBatchSizeBytes) func CloneReadBatchSizeBytes() int32 { - batchSizeBytes, _ := humanize.ParseBytes(os.Getenv("PCSM_CLONE_READ_BATCH_SIZE")) + sizeStr := viper.GetString("clone-read-batch-size") + if sizeStr == "" { + return 0 + } + + batchSizeBytes, _ := humanize.ParseBytes(sizeStr) return int32(min(batchSizeBytes, math.MaxInt32)) //nolint:gosec } diff --git a/main.go b/main.go index 1daf03e..557d2fe 100644 --- a/main.go +++ b/main.go @@ -353,6 +353,25 @@ func main() { "Use collection-level bulk write instead of client bulk write") rootCmd.PersistentFlags().MarkHidden("use-collection-bulk-write") //nolint:errcheck + // Clone tuning options (hidden - for advanced tuning) + // NOTE: These are CLI/HTTP-only, NO env var support per stakeholder decision #2 + rootCmd.PersistentFlags().Int("clone-num-parallel-collections", 0, + "Number of collections to clone in parallel (0 = auto)") + rootCmd.PersistentFlags().Int("clone-num-read-workers", 0, + "Number of read workers during clone (0 = auto)") + rootCmd.PersistentFlags().Int("clone-num-insert-workers", 0, + "Number of insert workers during clone (0 = auto)") + rootCmd.PersistentFlags().String("clone-segment-size", "", + "Segment size for clone operations (e.g., 100MB, 1GiB)") + rootCmd.PersistentFlags().String("clone-read-batch-size", "", + "Read batch size during clone (e.g., 16MiB)") + + rootCmd.PersistentFlags().MarkHidden("clone-num-parallel-collections") //nolint:errcheck + rootCmd.PersistentFlags().MarkHidden("clone-num-read-workers") //nolint:errcheck + rootCmd.PersistentFlags().MarkHidden("clone-num-insert-workers") //nolint:errcheck + rootCmd.PersistentFlags().MarkHidden("clone-segment-size") //nolint:errcheck + rootCmd.PersistentFlags().MarkHidden("clone-read-batch-size") //nolint:errcheck + statusCmd.Flags().Int("port", DefaultServerPort, "Port number") startCmd.Flags().Int("port", DefaultServerPort, "Port number") From 230d7598568acc91c1257cf700af4a9c38f53abf Mon Sep 17 00:00:00 2001 From: Adnan Date: Tue, 23 Dec 2025 18:55:57 +0100 Subject: [PATCH 05/17] PCSM-219: Add validator/v10 and create validation package - Add github.com/go-playground/validator/v10 dependency - Create validate/validate.go with singleton validator - Create validate/errors.go with error types and translation - Create validate/bytesize.go with custom byte size validators --- go.mod | 13 +++++--- go.sum | 28 +++++++++++----- validate/bytesize.go | 74 +++++++++++++++++++++++++++++++++++++++++ validate/errors.go | 78 ++++++++++++++++++++++++++++++++++++++++++++ validate/validate.go | 49 ++++++++++++++++++++++++++++ 5 files changed, 230 insertions(+), 12 deletions(-) create mode 100644 validate/bytesize.go create mode 100644 validate/errors.go create mode 100644 validate/validate.go diff --git a/go.mod b/go.mod index 05456ce..6f29ab1 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.25.0 require ( github.com/dustin/go-humanize v1.0.1 + github.com/go-playground/validator/v10 v10.30.0 github.com/prometheus/client_golang v1.22.0 github.com/rs/zerolog v1.34.0 github.com/spf13/cobra v1.9.1 @@ -11,7 +12,7 @@ require ( github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 go.mongodb.org/mongo-driver/v2 v2.2.1 - golang.org/x/sync v0.18.0 + golang.org/x/sync v0.19.0 ) require ( @@ -19,10 +20,14 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.12 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/compress v1.18.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -41,9 +46,9 @@ require ( github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect google.golang.org/protobuf v1.36.5 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 30eb651..c101f33 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,16 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= +github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.0 h1:5YBPNs273uzsZJD1I8uiB4Aqg9sN6sMDVX3s6LxmhWU= +github.com/go-playground/validator/v10 v10.30.0/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -29,6 +39,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -89,16 +101,16 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -107,16 +119,16 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/validate/bytesize.go b/validate/bytesize.go new file mode 100644 index 0000000..54f59a3 --- /dev/null +++ b/validate/bytesize.go @@ -0,0 +1,74 @@ +package validate + +import ( + "reflect" + + "github.com/dustin/go-humanize" + "github.com/go-playground/validator/v10" +) + +// validateByteSize checks if a string can be parsed as a byte size. +func validateByteSize(fl validator.FieldLevel) bool { + s := getStringValue(fl.Field()) + if s == "" || s == "0" { + return true // empty/zero = use default + } + + _, err := humanize.ParseBytes(s) + + return err == nil +} + +// validateByteSizeMin validates minimum byte size. +// Tag usage: bytesizemin=16MB +func validateByteSizeMin(fl validator.FieldLevel) bool { + s := getStringValue(fl.Field()) + if s == "" || s == "0" { + return true // empty/zero bypasses validation (use default) + } + + bytes, err := humanize.ParseBytes(s) + if err != nil { + return false + } + + minBytes, err := humanize.ParseBytes(fl.Param()) + if err != nil { + return false + } + + return bytes >= minBytes +} + +// validateByteSizeMax validates maximum byte size. +// Tag usage: bytesizemax=64GiB +func validateByteSizeMax(fl validator.FieldLevel) bool { + s := getStringValue(fl.Field()) + if s == "" || s == "0" { + return true + } + + bytes, err := humanize.ParseBytes(s) + if err != nil { + return false + } + + maxBytes, err := humanize.ParseBytes(fl.Param()) + if err != nil { + return false + } + + return bytes <= maxBytes +} + +func getStringValue(field reflect.Value) string { + if field.Kind() == reflect.Ptr { + if field.IsNil() { + return "" + } + + return field.Elem().String() + } + + return field.String() +} diff --git a/validate/errors.go b/validate/errors.go new file mode 100644 index 0000000..5c58d24 --- /dev/null +++ b/validate/errors.go @@ -0,0 +1,78 @@ +package validate + +import ( + "fmt" + "strings" + + "github.com/go-playground/validator/v10" +) + +// ValidationError represents a single validation error. +type ValidationError struct { + Field string `json:"field"` + Message string `json:"message"` +} + +func (e ValidationError) Error() string { + return fmt.Sprintf("%s: %s", e.Field, e.Message) +} + +// ValidationErrors is a collection of validation errors. +type ValidationErrors []ValidationError + +func (e ValidationErrors) Error() string { + var sb strings.Builder + for i, err := range e { + if i > 0 { + sb.WriteString("; ") + } + sb.WriteString(err.Error()) + } + + return sb.String() +} + +// TranslateErrors converts validator.ValidationErrors to user-friendly messages. +func TranslateErrors(err error) error { + if err == nil { + return nil + } + + validationErrors, ok := err.(validator.ValidationErrors) //nolint:errorlint + if !ok { + return err + } + + var errs ValidationErrors + for _, e := range validationErrors { + errs = append(errs, ValidationError{ + Field: e.Field(), + Message: translateFieldError(e), + }) + } + + return errs +} + +func translateFieldError(e validator.FieldError) string { + switch e.Tag() { + case "required": + return "is required" + case "gte": + return fmt.Sprintf("must be at least %s", e.Param()) + case "lte": + return fmt.Sprintf("must be at most %s", e.Param()) + case "min": + return fmt.Sprintf("must be at least %s", e.Param()) + case "max": + return fmt.Sprintf("must be at most %s", e.Param()) + case "bytesize": + return "must be a valid byte size (e.g., '100MB', '1GiB')" + case "bytesizemin": + return fmt.Sprintf("must be at least %s", e.Param()) + case "bytesizemax": + return fmt.Sprintf("must be at most %s", e.Param()) + default: + return fmt.Sprintf("failed %s validation", e.Tag()) + } +} diff --git a/validate/validate.go b/validate/validate.go new file mode 100644 index 0000000..d7e7f25 --- /dev/null +++ b/validate/validate.go @@ -0,0 +1,49 @@ +// Package validate provides request validation using go-playground/validator. +package validate + +import ( + "reflect" + "strings" + "sync" + + "github.com/go-playground/validator/v10" +) + +var ( + instance *validator.Validate //nolint:gochecknoglobals + once sync.Once //nolint:gochecknoglobals +) + +// Validator returns the singleton validator instance. +func Validator() *validator.Validate { + once.Do(func() { + instance = validator.New(validator.WithRequiredStructEnabled()) + registerCustomValidators(instance) + registerTagNameFunc(instance) + }) + + return instance +} + +func registerCustomValidators(v *validator.Validate) { + _ = v.RegisterValidation("bytesize", validateByteSize) + _ = v.RegisterValidation("bytesizemin", validateByteSizeMin) + _ = v.RegisterValidation("bytesizemax", validateByteSizeMax) +} + +// registerTagNameFunc uses JSON tag names in error messages. +func registerTagNameFunc(v *validator.Validate) { + v.RegisterTagNameFunc(func(fld reflect.StructField) string { + name := strings.SplitN(fld.Tag.Get("json"), ",", 2)[0] + if name == "-" { + return fld.Name + } + + return name + }) +} + +// Struct validates a struct using the singleton validator. +func Struct(s any) error { + return TranslateErrors(Validator().Struct(s)) +} From 7b414629ba4c601631e27bb48668709980ebd905 Mon Sep 17 00:00:00 2001 From: Adnan Date: Tue, 23 Dec 2025 18:57:10 +0100 Subject: [PATCH 06/17] PCSM-219: Add config/config.go --- config/config.go | 73 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 config/config.go diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..c869179 --- /dev/null +++ b/config/config.go @@ -0,0 +1,73 @@ +// Package config provides configuration management for PCSM using Viper. +package config + +import ( + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// Init initializes Viper configuration binding with Cobra command flags. +// This should be called in PersistentPreRun to ensure all flags are bound before use. +func Init(cmd *cobra.Command) { + // Set environment variable prefix + viper.SetEnvPrefix("PCSM") + + // Replace hyphens with underscores for env var lookup + // e.g., "log-level" -> "PCSM_LOG_LEVEL" + viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + + // Automatically read matching env vars + viper.AutomaticEnv() + + // Bind CLI flags to Viper + if cmd.PersistentFlags() != nil { + _ = viper.BindPFlags(cmd.PersistentFlags()) + } + if cmd.Flags() != nil { + _ = viper.BindPFlags(cmd.Flags()) + } + + // Bind specific env var names + bindEnvVars() +} + +// bindEnvVars binds environment variable names to Viper keys. +// Only global/server options have env var support. +// Clone tuning options are intentionally NOT bound (CLI/HTTP only). +func bindEnvVars() { + // Server connection URIs + _ = viper.BindEnv("source", "PCSM_SOURCE_URI") + _ = viper.BindEnv("target", "PCSM_TARGET_URI") + + // MongoDB client timeout + _ = viper.BindEnv("mongodb-cli-operation-timeout", "PCSM_MONGODB_CLI_OPERATION_TIMEOUT") + + // Bulk write option (internal, has env var support) + _ = viper.BindEnv("use-collection-bulk-write", "PCSM_USE_COLLECTION_BULK_WRITE") + + // NOTE: Clone tuning options intentionally NOT bound to env vars. + // They are CLI/HTTP-only per stakeholder decision (PCSM-219). + // See: comment-decisions.md - Decision #2 +} + +// GetString returns a string configuration value from Viper. +func GetString(key string) string { + return viper.GetString(key) +} + +// GetInt returns an integer configuration value from Viper. +func GetInt(key string) int { + return viper.GetInt(key) +} + +// GetBool returns a boolean configuration value from Viper. +func GetBool(key string) bool { + return viper.GetBool(key) +} + +// IsSet checks if a configuration key has been set. +func IsSet(key string) bool { + return viper.IsSet(key) +} From fe51882560f20bcc88f180a07966a127c68eefda Mon Sep 17 00:00:00 2001 From: Adnan Date: Tue, 23 Dec 2025 18:59:03 +0100 Subject: [PATCH 07/17] PCSM-219: Extend StartOptions and HTTP startRequest with validation - Extend pcsm.StartOptions with clone tuning fields - Update startRequest with clone tuning fields and validation tags - Add Validate() method to startRequest - Add resolveStartOptions() to merge HTTP/CLI/config values - Add resolveCloneSegmentSize/resolveCloneReadBatchSize helpers - HTTP values take precedence over CLI values --- main.go | 139 +++++++++++++++++++++++++++++++++++++++++++++++++-- pcsm/pcsm.go | 15 ++++++ 2 files changed, 149 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index 557d2fe..2e587e6 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "io" + "math" "net/http" "os" "os/signal" @@ -30,6 +31,7 @@ import ( "github.com/percona/percona-clustersync-mongodb/pcsm" "github.com/percona/percona-clustersync-mongodb/topo" "github.com/percona/percona-clustersync-mongodb/util" + "github.com/percona/percona-clustersync-mongodb/validate" ) // Constants for server configuration. @@ -734,6 +736,106 @@ func (s *server) handleStatus(w http.ResponseWriter, r *http.Request) { writeResponse(w, res) } +// resolveStartOptions resolves the start options from the HTTP request and config. +// HTTP request values take precedence over CLI flag values. +func resolveStartOptions(params startRequest) (*pcsm.StartOptions, error) { + options := &pcsm.StartOptions{ + PauseOnInitialSync: params.PauseOnInitialSync, + IncludeNamespaces: params.IncludeNamespaces, + ExcludeNamespaces: params.ExcludeNamespaces, + } + + // Clone parallelism: HTTP > CLI > default + if params.CloneNumParallelCollections != nil { + options.CloneParallelism = *params.CloneNumParallelCollections + } else { + options.CloneParallelism = config.CloneNumParallelCollections() + } + + // Clone read workers: HTTP > CLI > default + if params.CloneNumReadWorkers != nil { + options.CloneReadWorkers = *params.CloneNumReadWorkers + } else { + options.CloneReadWorkers = config.CloneNumReadWorkers() + } + + // Clone insert workers: HTTP > CLI > default + if params.CloneNumInsertWorkers != nil { + options.CloneInsertWorkers = *params.CloneNumInsertWorkers + } else { + options.CloneInsertWorkers = config.CloneNumInsertWorkers() + } + + // Clone segment size: HTTP > CLI > default + segmentSize, err := resolveCloneSegmentSize(params.CloneSegmentSize) + if err != nil { + return nil, err + } + options.CloneSegmentSizeBytes = segmentSize + + // Clone read batch size: HTTP > CLI > default + batchSize, err := resolveCloneReadBatchSize(params.CloneReadBatchSize) + if err != nil { + return nil, err + } + options.CloneReadBatchSizeBytes = batchSize + + // UseCollectionBulkWrite: internal only, always from config (CLI + env var via Viper) + options.UseCollectionBulkWrite = config.UseCollectionBulkWrite() + + return options, nil +} + +// resolveCloneSegmentSize resolves the clone segment size from HTTP or CLI. +func resolveCloneSegmentSize(value *string) (int64, error) { + if value != nil { + sizeBytes, err := humanize.ParseBytes(*value) + if err != nil { + return 0, errors.Wrapf(err, "invalid cloneSegmentSize value: %s", *value) + } + // Allow 0 (auto) or validate against min/max + if sizeBytes > 0 && sizeBytes < config.MinCloneSegmentSizeBytes { + return 0, errors.Errorf("cloneSegmentSize must be at least %s, got %s", + humanize.Bytes(config.MinCloneSegmentSizeBytes), + humanize.Bytes(sizeBytes)) + } + if sizeBytes > config.MaxCloneSegmentSizeBytes { + return 0, errors.Errorf("cloneSegmentSize must be at most %s, got %s", + humanize.Bytes(config.MaxCloneSegmentSizeBytes), + humanize.Bytes(sizeBytes)) + } + + return int64(min(sizeBytes, math.MaxInt64)), nil //nolint:gosec + } + + return config.CloneSegmentSizeBytes(), nil +} + +// resolveCloneReadBatchSize resolves the clone read batch size from HTTP or CLI. +func resolveCloneReadBatchSize(value *string) (int32, error) { + if value != nil { + sizeBytes, err := humanize.ParseBytes(*value) + if err != nil { + return 0, errors.Wrapf(err, "invalid cloneReadBatchSize value: %s", *value) + } + // Allow 0 (auto) or validate against min/max + if sizeBytes > 0 && sizeBytes < uint64(config.MinCloneReadBatchSizeBytes) { + return 0, errors.Errorf("cloneReadBatchSize must be at least %s, got %s", + humanize.Bytes(uint64(config.MinCloneReadBatchSizeBytes)), + humanize.Bytes(sizeBytes)) + } + if sizeBytes > uint64(config.MaxCloneReadBatchSizeBytes) { + return 0, errors.Errorf("cloneReadBatchSize must be at most %s, got %s", + humanize.Bytes(uint64(config.MaxCloneReadBatchSizeBytes)), + humanize.Bytes(sizeBytes)) + } + + return int32(min(sizeBytes, math.MaxInt32)), nil //nolint:gosec + } + + return config.CloneReadBatchSizeBytes(), nil +} + // handleStart handles the /start endpoint. func (s *server) handleStart(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), ServerResponseTimeout) @@ -777,13 +879,21 @@ func (s *server) handleStart(w http.ResponseWriter, r *http.Request) { } } - options := &pcsm.StartOptions{ - PauseOnInitialSync: params.PauseOnInitialSync, - IncludeNamespaces: params.IncludeNamespaces, - ExcludeNamespaces: params.ExcludeNamespaces, + // Validate request + if err := params.Validate(); err != nil { + writeResponse(w, startResponse{Err: err.Error()}) + + return } - err := s.pcsm.Start(ctx, options) + options, err := resolveStartOptions(params) + if err != nil { + writeResponse(w, startResponse{Err: err.Error()}) + + return + } + + err = s.pcsm.Start(ctx, options) if err != nil { writeResponse(w, startResponse{Err: err.Error()}) @@ -961,6 +1071,25 @@ type startRequest struct { IncludeNamespaces []string `json:"includeNamespaces,omitempty"` // ExcludeNamespaces are the namespaces to exclude from the replication. ExcludeNamespaces []string `json:"excludeNamespaces,omitempty"` + + // Clone tuning options (pointer types to distinguish "not set" from zero value) + // CloneNumParallelCollections is the number of collections to clone in parallel. + CloneNumParallelCollections *int `json:"cloneNumParallelCollections,omitempty" validate:"omitempty,gte=0,lte=100"` + // CloneNumReadWorkers is the number of read workers during clone. + CloneNumReadWorkers *int `json:"cloneNumReadWorkers,omitempty" validate:"omitempty,gte=0,lte=1000"` + // CloneNumInsertWorkers is the number of insert workers during clone. + CloneNumInsertWorkers *int `json:"cloneNumInsertWorkers,omitempty" validate:"omitempty,gte=0,lte=1000"` + // CloneSegmentSize is the segment size for clone operations (e.g., "100MB", "1GiB"). + CloneSegmentSize *string `json:"cloneSegmentSize,omitempty" validate:"omitempty,bytesize"` + // CloneReadBatchSize is the read batch size during clone (e.g., "16MiB"). + CloneReadBatchSize *string `json:"cloneReadBatchSize,omitempty" validate:"omitempty,bytesize"` + + // NOTE: UseCollectionBulkWrite intentionally NOT exposed via HTTP (internal only) +} + +// Validate validates the startRequest. +func (r *startRequest) Validate() error { + return validate.Struct(r) } // startResponse represents the response body for the /start endpoint. diff --git a/pcsm/pcsm.go b/pcsm/pcsm.go index 84952fa..b6b91f7 100644 --- a/pcsm/pcsm.go +++ b/pcsm/pcsm.go @@ -289,6 +289,21 @@ type StartOptions struct { IncludeNamespaces []string // ExcludeNamespaces are the namespaces to exclude. ExcludeNamespaces []string + + // Clone tuning options + // CloneParallelism is the number of collections to clone in parallel. + CloneParallelism int + // CloneReadWorkers is the number of read workers during clone. + CloneReadWorkers int + // CloneInsertWorkers is the number of insert workers during clone. + CloneInsertWorkers int + // CloneSegmentSizeBytes is the segment size for clone operations in bytes. + CloneSegmentSizeBytes int64 + // CloneReadBatchSizeBytes is the read batch size during clone in bytes. + CloneReadBatchSizeBytes int32 + + // UseCollectionBulkWrite indicates whether to use collection-level bulk write. + UseCollectionBulkWrite bool } // Start starts the replication process with the given options. From 74b3a8904d2e5630a038167802aad0534a9d193a Mon Sep 17 00:00:00 2001 From: Adnan Date: Tue, 23 Dec 2025 19:00:01 +0100 Subject: [PATCH 08/17] PCSM-219: Update README with configuration documentation - Add --mongodb-cli-operation-timeout to PCSM Options - Update Environment Variables section with table format - Add Clone Tuning Options section with CLI/HTTP parameters - Update HTTP API /start with clone tuning parameters documentation --- README.md | 72 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7430426..c877615 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,7 @@ When starting the PCSM server, you can use the following options: - `--log-level`: The log level (default: "info") - `--log-json`: Output log in JSON format with disabled color - `--no-color`: Disable log ASCI color +- `--mongodb-cli-operation-timeout`: Timeout for MongoDB operations (e.g., `30s`, `5m`) Example: @@ -160,14 +161,51 @@ bin/pcsm \ --target \ --port 2242 \ --log-level debug \ - --log-json + --log-json \ + --mongodb-cli-operation-timeout 10m ``` ## Environment Variables -- `PCSM_SOURCE_URI`: MongoDB connection string for the source cluster. -- `PCSM_TARGET_URI`: MongoDB connection string for the target cluster. -- `PCSM_MONGODB_CLI_OPERATION_TIMEOUT`: Timeout for MongoDB client operations; accepts Go durations like `30s`, `2m`, `1h` (default: `5m`). +The following environment variables are supported: + +| Option | Env Var | Default | +|---------------------------|--------------------------------------|---------| +| Source URI | `PCSM_SOURCE_URI` | - | +| Target URI | `PCSM_TARGET_URI` | - | +| Port | `PCSM_PORT` | 2242 | +| Log Level | `PCSM_LOG_LEVEL` | info | +| Log JSON | `PCSM_LOG_JSON` | false | +| No Color | `PCSM_NO_COLOR` | false | +| MongoDB Timeout | `PCSM_MONGODB_CLI_OPERATION_TIMEOUT` | 5m | +| Use Collection Bulk Write | `PCSM_USE_COLLECTION_BULK_WRITE` | false | + +> **Note**: Clone tuning options (see below) are intentionally NOT supported via environment variables. They are configurable via CLI flags and HTTP request parameters only. + +## Clone Tuning Options + +Advanced tuning options for the clone process. These are available via CLI flags +and HTTP request parameters, but NOT via environment variables. + +| CLI Flag | HTTP Parameter | Default | Description | +|------------------------------------|-------------------------------|---------|------------------------------------------| +| `--clone-num-parallel-collections` | `cloneNumParallelCollections` | 2 | Collections to clone in parallel (0-100) | +| `--clone-num-read-workers` | `cloneNumReadWorkers` | auto | Read workers during clone (0-1000) | +| `--clone-num-insert-workers` | `cloneNumInsertWorkers` | auto | Insert workers during clone (0-1000) | +| `--clone-segment-size` | `cloneSegmentSize` | auto | Segment size (min ~475MB, max 64GiB) | +| `--clone-read-batch-size` | `cloneReadBatchSize` | ~47.5MB | Read batch size (16MiB - 2GiB) | + +> **Note**: These CLI flags are hidden from `--help` output. They are intended for advanced tuning only. + +Example CLI usage: + +```sh +bin/pcsm \ + --source \ + --target \ + --clone-num-parallel-collections 8 \ + --clone-num-read-workers 16 +``` ## Log JSON Fields @@ -206,10 +244,19 @@ Starts the replication process. #### Request Body -- `includeNamespaces` (optional): List of namespaces to include in the replication. -- `excludeNamespaces` (optional): List of namespaces to exclude from the replication. +| Parameter | Type | Description | +|-------------------------------|----------|-------------------------------------------------| +| `includeNamespaces` | string[] | Namespaces to include in replication | +| `excludeNamespaces` | string[] | Namespaces to exclude from replication | +| `cloneNumParallelCollections` | int | Collections to clone in parallel (0-100) | +| `cloneNumReadWorkers` | int | Read workers during clone (0-1000) | +| `cloneNumInsertWorkers` | int | Insert workers during clone (0-1000) | +| `cloneSegmentSize` | string | Segment size (e.g., "500MB", "1GiB") | +| `cloneReadBatchSize` | string | Read batch size (e.g., "32MiB") | -Example: +> **Note**: HTTP request values take precedence over CLI flag values for clone tuning options. + +Example (basic): ```json { @@ -218,6 +265,17 @@ Example: } ``` +Example (with clone tuning): + +```json +{ + "includeNamespaces": ["mydb.*"], + "cloneNumParallelCollections": 8, + "cloneNumReadWorkers": 16, + "cloneSegmentSize": "500MB" +} +``` + #### Response - `ok`: Boolean indicating if the operation was successful. From 9ef4de1d0562ce8dc16a8ab6ed8b4e9ae02663fd Mon Sep 17 00:00:00 2001 From: Adnan Date: Tue, 23 Dec 2025 20:16:53 +0100 Subject: [PATCH 09/17] Fix .vscode/settings.json formatting conflict --- .vscode/settings.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 84f4acf..4a7b1a3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -23,7 +23,6 @@ "go.formatFlags": [ "-extra" ], - "go.formatTool": "gofumpt", "go.lintTool": "golangci-lint-v2", "go.useLanguageServer": true, "gopls": { From e358ba14734bdcf988a5330c53146c0e55bded3c Mon Sep 17 00:00:00 2001 From: Adnan Date: Tue, 23 Dec 2025 22:01:25 +0100 Subject: [PATCH 10/17] PCSM-219: Simplify port flag handling with Viper - Make --port a persistent flag (inherited by all subcommands) - Remove duplicate port flag definitions from subcommands - Simplify getPort() to use Viper directly (handles CLI > env var > default) - Remove unused pflag import --- main.go | 41 +++++++++++------------------------------ 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/main.go b/main.go index 2e587e6..8a2237f 100644 --- a/main.go +++ b/main.go @@ -19,7 +19,6 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/zerolog" "github.com/spf13/cobra" - "github.com/spf13/pflag" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/v2/mongo" "go.mongodb.org/mongo-driver/v2/x/mongo/driver/connstring" @@ -87,7 +86,7 @@ var rootCmd = &cobra.Command{ return nil } - port := getPort(cmd.Flags()) + port := getPort() // Use Viper to get source/target URIs (supports CLI flags and env vars) sourceURI := viper.GetString("source") @@ -148,7 +147,7 @@ var statusCmd = &cobra.Command{ Use: "status", Short: "Get the status of the replication process", RunE: func(cmd *cobra.Command, _ []string) error { - port := getPort(cmd.Flags()) + port := getPort() return NewClient(port).Status(cmd.Context()) }, @@ -159,7 +158,7 @@ var startCmd = &cobra.Command{ Use: "start", Short: "Start Cluster Replication", RunE: func(cmd *cobra.Command, _ []string) error { - port := getPort(cmd.Flags()) + port := getPort() pauseOnInitialSync, _ := cmd.Flags().GetBool("pause-on-initial-sync") includeNamespaces, _ := cmd.Flags().GetStringSlice("include-namespaces") @@ -180,7 +179,7 @@ var finalizeCmd = &cobra.Command{ Use: "finalize", Short: "Finalize Cluster Replication", RunE: func(cmd *cobra.Command, _ []string) error { - port := getPort(cmd.Flags()) + port := getPort() ignoreHistoryLost, _ := cmd.Flags().GetBool("ignore-history-lost") @@ -197,7 +196,7 @@ var pauseCmd = &cobra.Command{ Use: "pause", Short: "Pause Cluster Replication", RunE: func(cmd *cobra.Command, _ []string) error { - port := getPort(cmd.Flags()) + port := getPort() return NewClient(port).Pause(cmd.Context()) }, @@ -208,7 +207,7 @@ var resumeCmd = &cobra.Command{ Use: "resume", Short: "Resume Cluster Replication", RunE: func(cmd *cobra.Command, _ []string) error { - port := getPort(cmd.Flags()) + port := getPort() fromFailure, _ := cmd.Flags().GetBool("from-failure") @@ -313,21 +312,10 @@ var resetHeartbeatCmd = &cobra.Command{ }, } -func getPort(flags *pflag.FlagSet) int { - // First check if the flag was explicitly set on command line - if flags.Changed("port") { - port, _ := flags.GetInt("port") - - return port - } - - // Use Viper which handles env vars automatically (PCSM_PORT) - port := viper.GetInt("port") - if port == 0 { - port = DefaultServerPort - } - - return port +// getPort returns the port number from Viper configuration. +// Viper handles precedence: CLI flag > env var (PCSM_PORT) > default. +func getPort() int { + return viper.GetInt("port") } func main() { @@ -335,7 +323,7 @@ func main() { rootCmd.PersistentFlags().Bool("log-json", false, "Output log in JSON format") rootCmd.PersistentFlags().Bool("no-color", false, "Disable log color") - rootCmd.Flags().Int("port", DefaultServerPort, "Port number") + rootCmd.PersistentFlags().Int("port", DefaultServerPort, "Port number") rootCmd.Flags().String("source", "", "MongoDB connection string for the source") rootCmd.Flags().String("target", "", "MongoDB connection string for the target") rootCmd.Flags().Bool("start", false, "Start Cluster Replication immediately") @@ -374,9 +362,6 @@ func main() { rootCmd.PersistentFlags().MarkHidden("clone-segment-size") //nolint:errcheck rootCmd.PersistentFlags().MarkHidden("clone-read-batch-size") //nolint:errcheck - statusCmd.Flags().Int("port", DefaultServerPort, "Port number") - - startCmd.Flags().Int("port", DefaultServerPort, "Port number") startCmd.Flags().Bool("pause-on-initial-sync", false, "Pause on Initial Sync") startCmd.Flags().MarkHidden("pause-on-initial-sync") //nolint:errcheck startCmd.Flags().StringSlice("include-namespaces", nil, @@ -384,12 +369,8 @@ func main() { startCmd.Flags().StringSlice("exclude-namespaces", nil, "Namespaces to exclude from the replication (e.g. db3.collection3,db4.*)") - pauseCmd.Flags().Int("port", DefaultServerPort, "Port number") - - resumeCmd.Flags().Int("port", DefaultServerPort, "Port number") resumeCmd.Flags().Bool("from-failure", false, "Reuse from failure") - finalizeCmd.Flags().Int("port", DefaultServerPort, "Port number") finalizeCmd.Flags().Bool("ignore-history-lost", false, "Ignore history lost error") finalizeCmd.Flags().MarkHidden("ignore-history-lost") //nolint:errcheck From cd7ffc1c89b7df674f6472b55b0d91057baa711c Mon Sep 17 00:00:00 2001 From: Adnan Date: Tue, 23 Dec 2025 22:03:08 +0100 Subject: [PATCH 11/17] PCSM-219: Consolidate configuration documentation - Add configuration reference table to config/config.go (single source of truth) - Remove redundant config source comments from config/values.go - Remove references to external decision documents - Simplify function comments to describe behavior, not configuration --- config/config.go | 28 +++++++++++++++++++++------- config/values.go | 42 ++++++------------------------------------ 2 files changed, 27 insertions(+), 43 deletions(-) diff --git a/config/config.go b/config/config.go index c869179..cfc6d2d 100644 --- a/config/config.go +++ b/config/config.go @@ -34,8 +34,26 @@ func Init(cmd *cobra.Command) { } // bindEnvVars binds environment variable names to Viper keys. -// Only global/server options have env var support. -// Clone tuning options are intentionally NOT bound (CLI/HTTP only). +// +// Configuration Reference: +// +// | Viper Key | CLI Flag | Env Var | Default | +// |--------------------------------|-----------------------------------|--------------------------------------|---------| +// | source | --source | PCSM_SOURCE_URI | - | +// | target | --target | PCSM_TARGET_URI | - | +// | port | --port | PCSM_PORT | 2242 | +// | log-level | --log-level | PCSM_LOG_LEVEL | info | +// | log-json | --log-json | PCSM_LOG_JSON | false | +// | no-color | --no-color | PCSM_NO_COLOR | false | +// | mongodb-cli-operation-timeout | --mongodb-cli-operation-timeout | PCSM_MONGODB_CLI_OPERATION_TIMEOUT | 5m | +// | use-collection-bulk-write | --use-collection-bulk-write | PCSM_USE_COLLECTION_BULK_WRITE | false | +// | clone-num-parallel-collections | --clone-num-parallel-collections | - | 0 | +// | clone-num-read-workers | --clone-num-read-workers | - | 0 | +// | clone-num-insert-workers | --clone-num-insert-workers | - | 0 | +// | clone-segment-size | --clone-segment-size | - | 0 | +// | clone-read-batch-size | --clone-read-batch-size | - | 0 | +// +// Note: Clone tuning options are CLI/HTTP only (no env var support). func bindEnvVars() { // Server connection URIs _ = viper.BindEnv("source", "PCSM_SOURCE_URI") @@ -44,12 +62,8 @@ func bindEnvVars() { // MongoDB client timeout _ = viper.BindEnv("mongodb-cli-operation-timeout", "PCSM_MONGODB_CLI_OPERATION_TIMEOUT") - // Bulk write option (internal, has env var support) + // Bulk write option (hidden, internal use) _ = viper.BindEnv("use-collection-bulk-write", "PCSM_USE_COLLECTION_BULK_WRITE") - - // NOTE: Clone tuning options intentionally NOT bound to env vars. - // They are CLI/HTTP-only per stakeholder decision (PCSM-219). - // See: comment-decisions.md - Decision #2 } // GetString returns a string configuration value from Viper. diff --git a/config/values.go b/config/values.go index ae86ac6..0826ab4 100644 --- a/config/values.go +++ b/config/values.go @@ -12,52 +12,31 @@ import ( ) // UseCollectionBulkWrite returns whether to use collection-level bulk write. -// This is an internal option, not exposed via HTTP API. -// -// Configuration sources (in order of precedence): -// - CLI flag: --use-collection-bulk-write (hidden) -// - Env var: PCSM_USE_COLLECTION_BULK_WRITE -// - Default: false +// Internal option, not exposed via HTTP API. func UseCollectionBulkWrite() bool { return viper.GetBool("use-collection-bulk-write") } // CloneNumParallelCollections returns the number of collections to clone in parallel. -// Configurable via CLI flag only (no env var support per decision #2). -// -// Configuration sources (in order of precedence): -// - CLI flag: --clone-num-parallel-collections -// - Default: 0 (auto) +// Returns 0 for auto-detection. func CloneNumParallelCollections() int { return viper.GetInt("clone-num-parallel-collections") } // CloneNumReadWorkers returns the number of read workers. -// Configurable via CLI flag only (no env var support per decision #2). -// -// Configuration sources (in order of precedence): -// - CLI flag: --clone-num-read-workers -// - Default: 0 (auto) +// Returns 0 for auto-detection. func CloneNumReadWorkers() int { return viper.GetInt("clone-num-read-workers") } // CloneNumInsertWorkers returns the number of insert workers. -// Configurable via CLI flag only (no env var support per decision #2). -// -// Configuration sources (in order of precedence): -// - CLI flag: --clone-num-insert-workers -// - Default: 0 (auto) +// Returns 0 for auto-detection. func CloneNumInsertWorkers() int { return viper.GetInt("clone-num-insert-workers") } // CloneSegmentSizeBytes returns the segment size in bytes. -// Configurable via CLI flag only (no env var support per decision #2). -// -// Configuration sources (in order of precedence): -// - CLI flag: --clone-segment-size -// - Default: AutoCloneSegmentSize (0 = auto) +// Returns 0 (AutoCloneSegmentSize) for auto-detection. func CloneSegmentSizeBytes() int64 { sizeStr := viper.GetString("clone-segment-size") if sizeStr == "" { @@ -73,11 +52,7 @@ func CloneSegmentSizeBytes() int64 { } // CloneReadBatchSizeBytes returns the read batch size in bytes. -// Configurable via CLI flag only (no env var support per decision #2). -// -// Configuration sources (in order of precedence): -// - CLI flag: --clone-read-batch-size -// - Default: 0 (uses MaxWriteBatchSizeBytes) +// Returns 0 to use MaxWriteBatchSizeBytes default. func CloneReadBatchSizeBytes() int32 { sizeStr := viper.GetString("clone-read-batch-size") if sizeStr == "" { @@ -112,11 +87,6 @@ func UseTargetClientCompressors() []string { } // MongoDBOperationTimeout returns the timeout for MongoDB client operations. -// -// Configuration sources (in order of precedence): -// - CLI flag: --mongodb-cli-operation-timeout -// - Env var: PCSM_MONGODB_CLI_OPERATION_TIMEOUT -// - Default: 5m func MongoDBOperationTimeout() time.Duration { timeoutStr := viper.GetString("mongodb-cli-operation-timeout") if timeoutStr != "" { From be654336a767ab254c6d4486012fce48f9296774 Mon Sep 17 00:00:00 2001 From: Adnan Date: Tue, 23 Dec 2025 22:03:53 +0100 Subject: [PATCH 12/17] PCSM-219: Remove external decision references from comments - Replace decision references with self-documenting comments - Comments now describe what/why, not where it was decided --- main.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index 8a2237f..6d33435 100644 --- a/main.go +++ b/main.go @@ -333,18 +333,16 @@ func main() { rootCmd.Flags().MarkHidden("reset-state") //nolint:errcheck rootCmd.Flags().MarkHidden("pause-on-initial-sync") //nolint:errcheck - // MongoDB client options (visible per stakeholder decision #3) + // MongoDB client timeout (visible: commonly needed for debugging) rootCmd.PersistentFlags().String("mongodb-cli-operation-timeout", "", "Timeout for MongoDB operations (e.g., 30s, 5m)") - // NOTE: NOT marking as hidden per stakeholder decision - // Internal option (hidden, has env var support per decision #5) + // Bulk write option (hidden: internal tuning) rootCmd.PersistentFlags().Bool("use-collection-bulk-write", false, "Use collection-level bulk write instead of client bulk write") rootCmd.PersistentFlags().MarkHidden("use-collection-bulk-write") //nolint:errcheck - // Clone tuning options (hidden - for advanced tuning) - // NOTE: These are CLI/HTTP-only, NO env var support per stakeholder decision #2 + // Clone tuning options (hidden: advanced tuning, CLI/HTTP only) rootCmd.PersistentFlags().Int("clone-num-parallel-collections", 0, "Number of collections to clone in parallel (0 = auto)") rootCmd.PersistentFlags().Int("clone-num-read-workers", 0, From 7547453f6d490fe25b1bb2daad2a4bf276c5f366 Mon Sep 17 00:00:00 2001 From: Adnan Date: Tue, 23 Dec 2025 22:16:26 +0100 Subject: [PATCH 13/17] PCSM-219: Remove unnecessary Viper wrappers - Remove getPort() wrapper, use viper.GetInt("port") directly - Remove unused GetString, GetInt, GetBool, IsSet wrappers from config/config.go - These wrappers added no value (pure pass-through with no added logic) --- config/config.go | 20 -------------------- main.go | 28 ++++++---------------------- 2 files changed, 6 insertions(+), 42 deletions(-) diff --git a/config/config.go b/config/config.go index cfc6d2d..479ff87 100644 --- a/config/config.go +++ b/config/config.go @@ -65,23 +65,3 @@ func bindEnvVars() { // Bulk write option (hidden, internal use) _ = viper.BindEnv("use-collection-bulk-write", "PCSM_USE_COLLECTION_BULK_WRITE") } - -// GetString returns a string configuration value from Viper. -func GetString(key string) string { - return viper.GetString(key) -} - -// GetInt returns an integer configuration value from Viper. -func GetInt(key string) int { - return viper.GetInt(key) -} - -// GetBool returns a boolean configuration value from Viper. -func GetBool(key string) bool { - return viper.GetBool(key) -} - -// IsSet checks if a configuration key has been set. -func IsSet(key string) bool { - return viper.IsSet(key) -} diff --git a/main.go b/main.go index 6d33435..323e5dc 100644 --- a/main.go +++ b/main.go @@ -86,7 +86,7 @@ var rootCmd = &cobra.Command{ return nil } - port := getPort() + port := viper.GetInt("port") // Use Viper to get source/target URIs (supports CLI flags and env vars) sourceURI := viper.GetString("source") @@ -147,9 +147,7 @@ var statusCmd = &cobra.Command{ Use: "status", Short: "Get the status of the replication process", RunE: func(cmd *cobra.Command, _ []string) error { - port := getPort() - - return NewClient(port).Status(cmd.Context()) + return NewClient(viper.GetInt("port")).Status(cmd.Context()) }, } @@ -158,8 +156,6 @@ var startCmd = &cobra.Command{ Use: "start", Short: "Start Cluster Replication", RunE: func(cmd *cobra.Command, _ []string) error { - port := getPort() - pauseOnInitialSync, _ := cmd.Flags().GetBool("pause-on-initial-sync") includeNamespaces, _ := cmd.Flags().GetStringSlice("include-namespaces") excludeNamespaces, _ := cmd.Flags().GetStringSlice("exclude-namespaces") @@ -170,7 +166,7 @@ var startCmd = &cobra.Command{ ExcludeNamespaces: excludeNamespaces, } - return NewClient(port).Start(cmd.Context(), startOptions) + return NewClient(viper.GetInt("port")).Start(cmd.Context(), startOptions) }, } @@ -179,15 +175,13 @@ var finalizeCmd = &cobra.Command{ Use: "finalize", Short: "Finalize Cluster Replication", RunE: func(cmd *cobra.Command, _ []string) error { - port := getPort() - ignoreHistoryLost, _ := cmd.Flags().GetBool("ignore-history-lost") finalizeOptions := finalizeRequest{ IgnoreHistoryLost: ignoreHistoryLost, } - return NewClient(port).Finalize(cmd.Context(), finalizeOptions) + return NewClient(viper.GetInt("port")).Finalize(cmd.Context(), finalizeOptions) }, } @@ -196,9 +190,7 @@ var pauseCmd = &cobra.Command{ Use: "pause", Short: "Pause Cluster Replication", RunE: func(cmd *cobra.Command, _ []string) error { - port := getPort() - - return NewClient(port).Pause(cmd.Context()) + return NewClient(viper.GetInt("port")).Pause(cmd.Context()) }, } @@ -207,15 +199,13 @@ var resumeCmd = &cobra.Command{ Use: "resume", Short: "Resume Cluster Replication", RunE: func(cmd *cobra.Command, _ []string) error { - port := getPort() - fromFailure, _ := cmd.Flags().GetBool("from-failure") resumeOptions := resumeRequest{ FromFailure: fromFailure, } - return NewClient(port).Resume(cmd.Context(), resumeOptions) + return NewClient(viper.GetInt("port")).Resume(cmd.Context(), resumeOptions) }, } @@ -312,12 +302,6 @@ var resetHeartbeatCmd = &cobra.Command{ }, } -// getPort returns the port number from Viper configuration. -// Viper handles precedence: CLI flag > env var (PCSM_PORT) > default. -func getPort() int { - return viper.GetInt("port") -} - func main() { rootCmd.PersistentFlags().String("log-level", "info", "Log level") rootCmd.PersistentFlags().Bool("log-json", false, "Output log in JSON format") From 7133a98d6bc339ee73df147e4b62f7e580e6b1c3 Mon Sep 17 00:00:00 2001 From: Adnan Date: Wed, 24 Dec 2025 10:02:47 +0100 Subject: [PATCH 14/17] PCSM-219: Move to mapstruct for config PCSM-219: Update VSCode settings and README with additional config options PCSM-219: Fix --- .vscode/settings.json | 28 +++++++- README.md | 51 ++++++++------- config/config.go | 73 ++++++++++++--------- config/schema.go | 94 +++++++++++++++++++++++++++ config/values.go | 107 ------------------------------ main.go | 148 ++++++++++++++++++++++-------------------- pcsm/clone.go | 20 ++++-- pcsm/copy_test.go | 6 +- pcsm/pcsm.go | 10 +-- pcsm/repl.go | 25 ++++--- tests/perf_test.go | 11 ++-- topo/connect.go | 7 +- validate/validate.go | 14 ++++ 13 files changed, 335 insertions(+), 259 deletions(-) create mode 100644 config/schema.go delete mode 100644 config/values.go diff --git a/.vscode/settings.json b/.vscode/settings.json index 4a7b1a3..e097931 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -39,5 +39,31 @@ "tests" ], "python.testing.pytestEnabled": true, - "python.testing.unittestEnabled": false + "python.testing.unittestEnabled": false, + "cSpell.words": [ + "bson", + "clustersync", + "cmdutil", + "codegen", + "colls", + "connstring", + "contextcheck", + "Debugf", + "errgroup", + "errorlint", + "Infof", + "keygen", + "mapstructure", + "nolint", + "opencode", + "pcsm", + "pipefail", + "readconcern", + "readpref", + "Warnf", + "wrapcheck", + "Wrapf", + "writeconcern", + "zerolog" + ] } diff --git a/README.md b/README.md index c877615..4b42d95 100644 --- a/README.md +++ b/README.md @@ -145,13 +145,15 @@ curl http://localhost:2242/status When starting the PCSM server, you can use the following options: -- `--port`: The port on which the server will listen (default: 2242) -- `--source`: The MongoDB connection string for the source cluster -- `--target`: The MongoDB connection string for the target cluster -- `--log-level`: The log level (default: "info") -- `--log-json`: Output log in JSON format with disabled color -- `--no-color`: Disable log ASCI color -- `--mongodb-cli-operation-timeout`: Timeout for MongoDB operations (e.g., `30s`, `5m`) +| Option | CLI Flag | Default | Description | +|----------------------------------|------------------------------------|---------|--------------------------------------| +| Port | `--port` | 2242 | Port on which the server listens | +| Source URI | `--source` | - | MongoDB connection string for source | +| Target URI | `--target` | - | MongoDB connection string for target | +| Log Level | `--log-level` | info | Log level (trace/debug/info/warn/error/fatal/panic) | +| Log JSON | `--log-json` | false | Output log in JSON format | +| No Color | `--no-color` | false | Disable log ASCII color | +| MongoDB Operation Timeout | `--mongodb-cli-operation-timeout` | 5m | Timeout for MongoDB operations | Example: @@ -169,16 +171,16 @@ bin/pcsm \ The following environment variables are supported: -| Option | Env Var | Default | -|---------------------------|--------------------------------------|---------| -| Source URI | `PCSM_SOURCE_URI` | - | -| Target URI | `PCSM_TARGET_URI` | - | -| Port | `PCSM_PORT` | 2242 | -| Log Level | `PCSM_LOG_LEVEL` | info | -| Log JSON | `PCSM_LOG_JSON` | false | -| No Color | `PCSM_NO_COLOR` | false | -| MongoDB Timeout | `PCSM_MONGODB_CLI_OPERATION_TIMEOUT` | 5m | -| Use Collection Bulk Write | `PCSM_USE_COLLECTION_BULK_WRITE` | false | +| Option | Env Var | Default | Description | +|----------------------------|--------------------------------------|---------|--------------------------------------| +| Source URI | `PCSM_SOURCE_URI` | - | MongoDB connection string for source | +| Target URI | `PCSM_TARGET_URI` | - | MongoDB connection string for target | +| Port | `PCSM_PORT` | 2242 | Port on which the server listens | +| Log Level | `PCSM_LOG_LEVEL` | info | Log level | +| Log JSON | `PCSM_LOG_JSON` | false | Output log in JSON format | +| No Color | `PCSM_NO_COLOR` | false | Disable log ASCII color | +| MongoDB Operation Timeout | `PCSM_MONGODB_CLI_OPERATION_TIMEOUT` | 5m | Timeout for MongoDB operations | +| Use Collection Bulk Write | `PCSM_USE_COLLECTION_BULK_WRITE` | false | Use collection-level bulk write (internal) | > **Note**: Clone tuning options (see below) are intentionally NOT supported via environment variables. They are configurable via CLI flags and HTTP request parameters only. @@ -187,15 +189,16 @@ The following environment variables are supported: Advanced tuning options for the clone process. These are available via CLI flags and HTTP request parameters, but NOT via environment variables. -| CLI Flag | HTTP Parameter | Default | Description | -|------------------------------------|-------------------------------|---------|------------------------------------------| -| `--clone-num-parallel-collections` | `cloneNumParallelCollections` | 2 | Collections to clone in parallel (0-100) | -| `--clone-num-read-workers` | `cloneNumReadWorkers` | auto | Read workers during clone (0-1000) | -| `--clone-num-insert-workers` | `cloneNumInsertWorkers` | auto | Insert workers during clone (0-1000) | -| `--clone-segment-size` | `cloneSegmentSize` | auto | Segment size (min ~475MB, max 64GiB) | -| `--clone-read-batch-size` | `cloneReadBatchSize` | ~47.5MB | Read batch size (16MiB - 2GiB) | +| CLI Flag | HTTP Parameter | Default | Range | Description | +|------------------------------------|-------------------------------|----------|-------------|------------------------------------| +| `--clone-num-parallel-collections` | `cloneNumParallelCollections` | 2 | 0-100 | Collections to clone in parallel | +| `--clone-num-read-workers` | `cloneNumReadWorkers` | auto (0) | 0-1000 | Read workers during clone | +| `--clone-num-insert-workers` | `cloneNumInsertWorkers` | auto (0) | 0-1000 | Insert workers during clone | +| `--clone-segment-size` | `cloneSegmentSize` | auto | ~475MB-64GB | Segment size for parallel cloning | +| `--clone-read-batch-size` | `cloneReadBatchSize` | ~47.5MB | 16MiB-2GiB | Read cursor batch size | > **Note**: These CLI flags are hidden from `--help` output. They are intended for advanced tuning only. +> Setting a value to 0 or empty string uses the automatic/default behavior. Example CLI usage: diff --git a/config/config.go b/config/config.go index 479ff87..c1539e3 100644 --- a/config/config.go +++ b/config/config.go @@ -2,26 +2,23 @@ package config import ( + "os" + "slices" "strings" "github.com/spf13/cobra" "github.com/spf13/viper" + + "github.com/percona/percona-clustersync-mongodb/errors" + "github.com/percona/percona-clustersync-mongodb/validate" ) -// Init initializes Viper configuration binding with Cobra command flags. -// This should be called in PersistentPreRun to ensure all flags are bound before use. -func Init(cmd *cobra.Command) { - // Set environment variable prefix +// Load initializes Viper and returns a validated Config. +func Load(cmd *cobra.Command) (*Config, error) { viper.SetEnvPrefix("PCSM") - - // Replace hyphens with underscores for env var lookup - // e.g., "log-level" -> "PCSM_LOG_LEVEL" viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) - - // Automatically read matching env vars viper.AutomaticEnv() - // Bind CLI flags to Viper if cmd.PersistentFlags() != nil { _ = viper.BindPFlags(cmd.PersistentFlags()) } @@ -29,30 +26,24 @@ func Init(cmd *cobra.Command) { _ = viper.BindPFlags(cmd.Flags()) } - // Bind specific env var names bindEnvVars() + + var cfg Config + + err := viper.Unmarshal(&cfg) + if err != nil { + return nil, errors.Wrap(err, "unmarshal config") + } + + err = validate.Struct(&cfg) + if err != nil { + return nil, errors.Wrap(err, "validate config") + } + + return &cfg, nil } // bindEnvVars binds environment variable names to Viper keys. -// -// Configuration Reference: -// -// | Viper Key | CLI Flag | Env Var | Default | -// |--------------------------------|-----------------------------------|--------------------------------------|---------| -// | source | --source | PCSM_SOURCE_URI | - | -// | target | --target | PCSM_TARGET_URI | - | -// | port | --port | PCSM_PORT | 2242 | -// | log-level | --log-level | PCSM_LOG_LEVEL | info | -// | log-json | --log-json | PCSM_LOG_JSON | false | -// | no-color | --no-color | PCSM_NO_COLOR | false | -// | mongodb-cli-operation-timeout | --mongodb-cli-operation-timeout | PCSM_MONGODB_CLI_OPERATION_TIMEOUT | 5m | -// | use-collection-bulk-write | --use-collection-bulk-write | PCSM_USE_COLLECTION_BULK_WRITE | false | -// | clone-num-parallel-collections | --clone-num-parallel-collections | - | 0 | -// | clone-num-read-workers | --clone-num-read-workers | - | 0 | -// | clone-num-insert-workers | --clone-num-insert-workers | - | 0 | -// | clone-segment-size | --clone-segment-size | - | 0 | -// | clone-read-batch-size | --clone-read-batch-size | - | 0 | -// // Note: Clone tuning options are CLI/HTTP only (no env var support). func bindEnvVars() { // Server connection URIs @@ -65,3 +56,25 @@ func bindEnvVars() { // Bulk write option (hidden, internal use) _ = viper.BindEnv("use-collection-bulk-write", "PCSM_USE_COLLECTION_BULK_WRITE") } + +// UseTargetClientCompressors returns a list of enabled compressors (from "zstd", "zlib", "snappy") +// for the target MongoDB client connection, as specified by the comma-separated environment +// variable PCSM_DEV_TARGET_CLIENT_COMPRESSORS. If unset or empty, returns nil. +func UseTargetClientCompressors() []string { + s := strings.TrimSpace(os.Getenv("PCSM_DEV_TARGET_CLIENT_COMPRESSORS")) + if s == "" { + return nil + } + + allowCompressors := []string{"zstd", "zlib", "snappy"} + + rv := make([]string, 0, min(len(s), len(allowCompressors))) + for a := range strings.SplitSeq(s, ",") { + a = strings.TrimSpace(a) + if slices.Contains(allowCompressors, a) && !slices.Contains(rv, a) { + rv = append(rv, a) + } + } + + return rv +} diff --git a/config/schema.go b/config/schema.go new file mode 100644 index 0000000..06401e0 --- /dev/null +++ b/config/schema.go @@ -0,0 +1,94 @@ +package config + +import ( + "math" + "time" + + "github.com/dustin/go-humanize" +) + +// Config holds all PCSM configuration. +type Config struct { + // Connection + Port int `mapstructure:"port" validate:"omitempty,gte=1024,lte=65535"` + Source string `mapstructure:"source"` + Target string `mapstructure:"target"` + + // Logging (squash keeps flat keys) + Log LogConfig `mapstructure:",squash"` + + // MongoDB client options + MongoDB MongoDBConfig `mapstructure:",squash"` + + // Clone tuning (CLI/HTTP only) + Clone CloneConfig `mapstructure:",squash"` + + // Internal options + UseCollectionBulkWrite bool `mapstructure:"use-collection-bulk-write"` + + // Hidden startup flags + Start bool `mapstructure:"start"` + ResetState bool `mapstructure:"reset-state"` + PauseOnInitialSync bool `mapstructure:"pause-on-initial-sync"` +} + +// LogConfig holds logging configuration. +type LogConfig struct { + Level string `mapstructure:"log-level" validate:"omitempty,oneof=trace debug info warn error fatal panic"` + JSON bool `mapstructure:"log-json"` + NoColor bool `mapstructure:"no-color"` +} + +// MongoDBConfig holds MongoDB client configuration. +type MongoDBConfig struct { + OperationTimeout string `mapstructure:"mongodb-cli-operation-timeout" validate:"omitempty,duration"` +} + +// OperationTimeoutDuration returns the parsed timeout or default. +func (m *MongoDBConfig) OperationTimeoutDuration() time.Duration { + if m.OperationTimeout != "" { + d, err := time.ParseDuration(m.OperationTimeout) + if err == nil && d > 0 { + return d + } + } + + return DefaultMongoDBCliOperationTimeout +} + +// CloneConfig holds clone tuning configuration. +type CloneConfig struct { + NumParallelCollections int `mapstructure:"clone-num-parallel-collections" validate:"omitempty,gte=0,lte=100"` + NumReadWorkers int `mapstructure:"clone-num-read-workers" validate:"omitempty,gte=0,lte=1000"` + NumInsertWorkers int `mapstructure:"clone-num-insert-workers" validate:"omitempty,gte=0,lte=1000"` + SegmentSize string `mapstructure:"clone-segment-size" validate:"omitempty,bytesize"` + ReadBatchSize string `mapstructure:"clone-read-batch-size" validate:"omitempty,bytesize"` +} + +// SegmentSizeBytes parses and returns the segment size in bytes. +func (c *CloneConfig) SegmentSizeBytes() int64 { + if c.SegmentSize == "" { + return AutoCloneSegmentSize + } + + bytes, err := humanize.ParseBytes(c.SegmentSize) + if err == nil && bytes > 0 { + return int64(min(bytes, math.MaxInt64)) //nolint:gosec + } + + return AutoCloneSegmentSize +} + +// ReadBatchSizeBytes parses and returns the read batch size in bytes. +func (c *CloneConfig) ReadBatchSizeBytes() int32 { + if c.ReadBatchSize == "" { + return 0 + } + + bytes, err := humanize.ParseBytes(c.ReadBatchSize) + if err == nil { + return int32(min(bytes, math.MaxInt32)) //nolint:gosec + } + + return 0 +} diff --git a/config/values.go b/config/values.go deleted file mode 100644 index 0826ab4..0000000 --- a/config/values.go +++ /dev/null @@ -1,107 +0,0 @@ -package config - -import ( - "math" - "os" - "slices" - "strings" - "time" - - "github.com/dustin/go-humanize" - "github.com/spf13/viper" -) - -// UseCollectionBulkWrite returns whether to use collection-level bulk write. -// Internal option, not exposed via HTTP API. -func UseCollectionBulkWrite() bool { - return viper.GetBool("use-collection-bulk-write") -} - -// CloneNumParallelCollections returns the number of collections to clone in parallel. -// Returns 0 for auto-detection. -func CloneNumParallelCollections() int { - return viper.GetInt("clone-num-parallel-collections") -} - -// CloneNumReadWorkers returns the number of read workers. -// Returns 0 for auto-detection. -func CloneNumReadWorkers() int { - return viper.GetInt("clone-num-read-workers") -} - -// CloneNumInsertWorkers returns the number of insert workers. -// Returns 0 for auto-detection. -func CloneNumInsertWorkers() int { - return viper.GetInt("clone-num-insert-workers") -} - -// CloneSegmentSizeBytes returns the segment size in bytes. -// Returns 0 (AutoCloneSegmentSize) for auto-detection. -func CloneSegmentSizeBytes() int64 { - sizeStr := viper.GetString("clone-segment-size") - if sizeStr == "" { - return AutoCloneSegmentSize - } - - segmentSizeBytes, _ := humanize.ParseBytes(sizeStr) - if segmentSizeBytes == 0 { - return AutoCloneSegmentSize - } - - return int64(min(segmentSizeBytes, math.MaxInt64)) //nolint:gosec -} - -// CloneReadBatchSizeBytes returns the read batch size in bytes. -// Returns 0 to use MaxWriteBatchSizeBytes default. -func CloneReadBatchSizeBytes() int32 { - sizeStr := viper.GetString("clone-read-batch-size") - if sizeStr == "" { - return 0 - } - - batchSizeBytes, _ := humanize.ParseBytes(sizeStr) - - return int32(min(batchSizeBytes, math.MaxInt32)) //nolint:gosec -} - -// UseTargetClientCompressors returns a list of enabled compressors (from "zstd", "zlib", "snappy") -// for the target MongoDB client connection, as specified by the comma-separated environment -// variable PCSM_DEV_TARGET_CLIENT_COMPRESSORS. If unset or empty, returns nil. -func UseTargetClientCompressors() []string { - s := strings.TrimSpace(os.Getenv("PCSM_DEV_TARGET_CLIENT_COMPRESSORS")) - if s == "" { - return nil - } - - allowCompressors := []string{"zstd", "zlib", "snappy"} - - rv := make([]string, 0, min(len(s), len(allowCompressors))) - for a := range strings.SplitSeq(s, ",") { - a = strings.TrimSpace(a) - if slices.Contains(allowCompressors, a) && !slices.Contains(rv, a) { - rv = append(rv, a) - } - } - - return rv -} - -// MongoDBOperationTimeout returns the timeout for MongoDB client operations. -func MongoDBOperationTimeout() time.Duration { - timeoutStr := viper.GetString("mongodb-cli-operation-timeout") - if timeoutStr != "" { - d, err := time.ParseDuration(timeoutStr) - if err == nil && d > 0 { - return d - } - } - - return DefaultMongoDBCliOperationTimeout -} - -// OperationMongoDBCliTimeout is an alias for MongoDBOperationTimeout for backward compatibility. -// -// Deprecated: Use MongoDBOperationTimeout instead. -func OperationMongoDBCliTimeout() time.Duration { - return MongoDBOperationTimeout() -} diff --git a/main.go b/main.go index 323e5dc..8435eee 100644 --- a/main.go +++ b/main.go @@ -42,6 +42,12 @@ const ( ServerResponseTimeout = 5 * time.Second ) +// contextKey is a type for context keys used in this package. +type contextKey string + +// configContextKey is the context key for storing *config.Config. +const configContextKey contextKey = "config" + var ( Version = "v0.6.0" //nolint:gochecknoglobals Platform = "" //nolint:gochecknoglobals @@ -61,23 +67,24 @@ var rootCmd = &cobra.Command{ SilenceUsage: true, - PersistentPreRun: func(cmd *cobra.Command, _ []string) { - // Initialize Viper config binding - config.Init(cmd) - - // Get logging configuration from Viper (supports CLI flags and env vars) - logLevelFlag := viper.GetString("log-level") - logJSON := viper.GetBool("log-json") - logNoColor := viper.GetBool("no-color") + PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { + // Load and validate config + cfg, err := config.Load(cmd) + if err != nil { + return errors.Wrap(err, "load config") + } - logLevel, err := zerolog.ParseLevel(logLevelFlag) + logLevel, err := zerolog.ParseLevel(cfg.Log.Level) if err != nil { - log.InitGlobals(0, logJSON, true).Fatal().Msg("Unknown log level") + logLevel = zerolog.InfoLevel } - lg := log.InitGlobals(logLevel, logJSON, logNoColor) + lg := log.InitGlobals(logLevel, cfg.Log.JSON, cfg.Log.NoColor) ctx := lg.WithContext(context.Background()) + ctx = context.WithValue(ctx, configContextKey, cfg) cmd.SetContext(ctx) + + return nil }, RunE: func(cmd *cobra.Command, _ []string) error { @@ -86,21 +93,18 @@ var rootCmd = &cobra.Command{ return nil } - port := viper.GetInt("port") + cfg := cmd.Context().Value(configContextKey).(*config.Config) //nolint:forcetypeassert - // Use Viper to get source/target URIs (supports CLI flags and env vars) - sourceURI := viper.GetString("source") - if sourceURI == "" { + if cfg.Source == "" { return errors.New("required flag --source not set") } - targetURI := viper.GetString("target") - if targetURI == "" { + if cfg.Target == "" { return errors.New("required flag --target not set") } - if ok, _ := cmd.Flags().GetBool("reset-state"); ok { - err := resetState(cmd.Context(), targetURI) + if cfg.ResetState { + err := resetState(cmd.Context(), cfg.Target, cfg) if err != nil { return err } @@ -108,18 +112,9 @@ var rootCmd = &cobra.Command{ log.New("cli").Info("State has been reset") } - start, _ := cmd.Flags().GetBool("start") - pause, _ := cmd.Flags().GetBool("pause-on-initial-sync") - log.Ctx(cmd.Context()).Info("Percona ClusterSync for MongoDB " + buildVersion()) - return runServer(cmd.Context(), serverOptions{ - port: port, - sourceURI: sourceURI, - targetURI: targetURI, - start: start, - pause: pause, - }) + return runServer(cmd.Context(), cfg) }, } @@ -214,12 +209,14 @@ var resetCmd = &cobra.Command{ Use: "reset", Short: "Reset PCSM state (heartbeat and recovery data)", RunE: func(cmd *cobra.Command, _ []string) error { + cfg := cmd.Context().Value(configContextKey).(*config.Config) //nolint:forcetypeassert + targetURI := viper.GetString("target") if targetURI == "" { return errors.New("required flag --target not set") } - err := resetState(cmd.Context(), targetURI) + err := resetState(cmd.Context(), targetURI, cfg) if err != nil { return err } @@ -236,6 +233,8 @@ var resetRecoveryCmd = &cobra.Command{ Hidden: true, Short: "Reset recovery state", RunE: func(cmd *cobra.Command, _ []string) error { + cfg := cmd.Context().Value(configContextKey).(*config.Config) //nolint:forcetypeassert + targetURI := viper.GetString("target") if targetURI == "" { return errors.New("required flag --target not set") @@ -243,7 +242,7 @@ var resetRecoveryCmd = &cobra.Command{ ctx := cmd.Context() - target, err := topo.Connect(ctx, targetURI) + target, err := topo.Connect(ctx, targetURI, cfg) if err != nil { return errors.Wrap(err, "connect") } @@ -272,6 +271,8 @@ var resetHeartbeatCmd = &cobra.Command{ Hidden: true, Short: "Reset heartbeat state", RunE: func(cmd *cobra.Command, _ []string) error { + cfg := cmd.Context().Value(configContextKey).(*config.Config) //nolint:forcetypeassert + targetURI := viper.GetString("target") if targetURI == "" { return errors.New("required flag --target not set") @@ -279,7 +280,7 @@ var resetHeartbeatCmd = &cobra.Command{ ctx := cmd.Context() - target, err := topo.Connect(ctx, targetURI) + target, err := topo.Connect(ctx, targetURI, cfg) if err != nil { return errors.Wrap(err, "connect") } @@ -375,8 +376,8 @@ func main() { } } -func resetState(ctx context.Context, targetURI string) error { - target, err := topo.Connect(ctx, targetURI) +func resetState(ctx context.Context, targetURI string, cfg *config.Config) error { + target, err := topo.Connect(ctx, targetURI, cfg) if err != nil { return errors.Wrap(err, "connect") } @@ -401,27 +402,24 @@ func resetState(ctx context.Context, targetURI string) error { return nil } -type serverOptions struct { - port int - sourceURI string - targetURI string - start bool - pause bool -} +func validateConfig(cfg *config.Config) error { + port := cfg.Port + if port == 0 { + port = DefaultServerPort + } -func (s serverOptions) validate() error { - if s.port <= 1024 || s.port > 65535 { + if port <= 1024 || port > 65535 { return errors.New("port value is outside the supported range [1024 - 65535]") } switch { - case s.sourceURI == "" && s.targetURI == "": + case cfg.Source == "" && cfg.Target == "": return errors.New("source URI and target URI are empty") - case s.sourceURI == "": + case cfg.Source == "": return errors.New("source URI is empty") - case s.targetURI == "": + case cfg.Target == "": return errors.New("target URI is empty") - case s.sourceURI == s.targetURI: + case cfg.Source == cfg.Target: return errors.New("source URI and target URI are identical") } @@ -429,8 +427,8 @@ func (s serverOptions) validate() error { } // runServer starts the HTTP server with the provided configuration. -func runServer(ctx context.Context, options serverOptions) error { - err := options.validate() +func runServer(ctx context.Context, cfg *config.Config) error { + err := validateConfig(cfg) if err != nil { return errors.Wrap(err, "validate options") } @@ -438,14 +436,14 @@ func runServer(ctx context.Context, options serverOptions) error { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) defer stop() - srv, err := createServer(ctx, options.sourceURI, options.targetURI) + srv, err := createServer(ctx, cfg) if err != nil { return errors.Wrap(err, "new server") } - if options.start && srv.pcsm.Status(ctx).State == pcsm.StateIdle { + if cfg.Start && srv.pcsm.Status(ctx).State == pcsm.StateIdle { err = srv.pcsm.Start(ctx, &pcsm.StartOptions{ - PauseOnInitialSync: options.pause, + PauseOnInitialSync: cfg.PauseOnInitialSync, }) if err != nil { log.New("cli").Error(err, "Failed to start Cluster Replication") @@ -463,7 +461,12 @@ func runServer(ctx context.Context, options serverOptions) error { os.Exit(0) }() - addr := fmt.Sprintf("localhost:%d", options.port) + port := cfg.Port + if port == 0 { + port = DefaultServerPort + } + + addr := fmt.Sprintf("localhost:%d", port) httpServer := http.Server{ Addr: addr, Handler: srv.Handler(), @@ -479,6 +482,8 @@ func runServer(ctx context.Context, options serverOptions) error { // server represents the replication server. type server struct { + // cfg holds the configuration. + cfg *config.Config // sourceCluster is the MongoDB client for the source cluster. sourceCluster *mongo.Client // targetCluster is the MongoDB client for the target cluster. @@ -493,10 +498,10 @@ type server struct { } // createServer creates a new server with the given options. -func createServer(ctx context.Context, sourceURI, targetURI string) (*server, error) { +func createServer(ctx context.Context, cfg *config.Config) (*server, error) { lg := log.Ctx(ctx) - source, err := topo.Connect(ctx, sourceURI) + source, err := topo.Connect(ctx, cfg.Source, cfg) if err != nil { return nil, errors.Wrap(err, "connect to source cluster") } @@ -517,11 +522,11 @@ func createServer(ctx context.Context, sourceURI, targetURI string) (*server, er return nil, errors.Wrap(err, "source version") } - cs, _ := connstring.Parse(sourceURI) + cs, _ := connstring.Parse(cfg.Source) lg.Infof("Connected to source cluster [%s]: %s://%s", sourceVersion.FullString(), cs.Scheme, strings.Join(cs.Hosts, ",")) - target, err := topo.ConnectWithOptions(ctx, targetURI, &topo.ConnectOptions{ + target, err := topo.ConnectWithOptions(ctx, cfg.Target, cfg, &topo.ConnectOptions{ Compressors: config.UseTargetClientCompressors(), }) if err != nil { @@ -544,7 +549,7 @@ func createServer(ctx context.Context, sourceURI, targetURI string) (*server, er return nil, errors.Wrap(err, "target version") } - cs, _ = connstring.Parse(targetURI) + cs, _ = connstring.Parse(cfg.Target) lg.Infof("Connected to target cluster [%s]: %s://%s", targetVersion.FullString(), cs.Scheme, strings.Join(cs.Hosts, ",")) @@ -575,6 +580,7 @@ func createServer(ctx context.Context, sourceURI, targetURI string) (*server, er go RunCheckpointing(ctx, target, pcs) s := &server{ + cfg: cfg, sourceCluster: source, targetCluster: target, pcsm: pcs, @@ -701,7 +707,7 @@ func (s *server) handleStatus(w http.ResponseWriter, r *http.Request) { // resolveStartOptions resolves the start options from the HTTP request and config. // HTTP request values take precedence over CLI flag values. -func resolveStartOptions(params startRequest) (*pcsm.StartOptions, error) { +func resolveStartOptions(cfg *config.Config, params startRequest) (*pcsm.StartOptions, error) { options := &pcsm.StartOptions{ PauseOnInitialSync: params.PauseOnInitialSync, IncludeNamespaces: params.IncludeNamespaces, @@ -712,45 +718,45 @@ func resolveStartOptions(params startRequest) (*pcsm.StartOptions, error) { if params.CloneNumParallelCollections != nil { options.CloneParallelism = *params.CloneNumParallelCollections } else { - options.CloneParallelism = config.CloneNumParallelCollections() + options.CloneParallelism = cfg.Clone.NumParallelCollections } // Clone read workers: HTTP > CLI > default if params.CloneNumReadWorkers != nil { options.CloneReadWorkers = *params.CloneNumReadWorkers } else { - options.CloneReadWorkers = config.CloneNumReadWorkers() + options.CloneReadWorkers = cfg.Clone.NumReadWorkers } // Clone insert workers: HTTP > CLI > default if params.CloneNumInsertWorkers != nil { options.CloneInsertWorkers = *params.CloneNumInsertWorkers } else { - options.CloneInsertWorkers = config.CloneNumInsertWorkers() + options.CloneInsertWorkers = cfg.Clone.NumInsertWorkers } // Clone segment size: HTTP > CLI > default - segmentSize, err := resolveCloneSegmentSize(params.CloneSegmentSize) + segmentSize, err := resolveCloneSegmentSize(cfg, params.CloneSegmentSize) if err != nil { return nil, err } options.CloneSegmentSizeBytes = segmentSize // Clone read batch size: HTTP > CLI > default - batchSize, err := resolveCloneReadBatchSize(params.CloneReadBatchSize) + batchSize, err := resolveCloneReadBatchSize(cfg, params.CloneReadBatchSize) if err != nil { return nil, err } options.CloneReadBatchSizeBytes = batchSize // UseCollectionBulkWrite: internal only, always from config (CLI + env var via Viper) - options.UseCollectionBulkWrite = config.UseCollectionBulkWrite() + options.UseCollectionBulkWrite = cfg.UseCollectionBulkWrite return options, nil } // resolveCloneSegmentSize resolves the clone segment size from HTTP or CLI. -func resolveCloneSegmentSize(value *string) (int64, error) { +func resolveCloneSegmentSize(cfg *config.Config, value *string) (int64, error) { if value != nil { sizeBytes, err := humanize.ParseBytes(*value) if err != nil { @@ -771,11 +777,12 @@ func resolveCloneSegmentSize(value *string) (int64, error) { return int64(min(sizeBytes, math.MaxInt64)), nil //nolint:gosec } - return config.CloneSegmentSizeBytes(), nil + // Fall back to CLI value + return cfg.Clone.SegmentSizeBytes(), nil } // resolveCloneReadBatchSize resolves the clone read batch size from HTTP or CLI. -func resolveCloneReadBatchSize(value *string) (int32, error) { +func resolveCloneReadBatchSize(cfg *config.Config, value *string) (int32, error) { if value != nil { sizeBytes, err := humanize.ParseBytes(*value) if err != nil { @@ -796,7 +803,8 @@ func resolveCloneReadBatchSize(value *string) (int32, error) { return int32(min(sizeBytes, math.MaxInt32)), nil //nolint:gosec } - return config.CloneReadBatchSizeBytes(), nil + // Fall back to CLI value + return cfg.Clone.ReadBatchSizeBytes(), nil } // handleStart handles the /start endpoint. @@ -849,7 +857,7 @@ func (s *server) handleStart(w http.ResponseWriter, r *http.Request) { return } - options, err := resolveStartOptions(params) + options, err := resolveStartOptions(s.cfg, params) if err != nil { writeResponse(w, startResponse{Err: err.Error()}) diff --git a/pcsm/clone.go b/pcsm/clone.go index 430b260..d3cc25c 100644 --- a/pcsm/clone.go +++ b/pcsm/clone.go @@ -28,6 +28,7 @@ type Clone struct { target *mongo.Client // Target MongoDB client catalog *Catalog // Catalog for managing collections and indexes nsFilter sel.NSFilter // Namespace filter + options *StartOptions // Clone options from StartOptions lock sync.Mutex err error // Error encountered during the cloning process @@ -74,12 +75,19 @@ func (cs *CloneStatus) IsFinished() bool { return !cs.FinishTime.IsZero() } -func NewClone(source, target *mongo.Client, catalog *Catalog, nsFilter sel.NSFilter) *Clone { +// NewClone creates a new Clone instance with the given options. +func NewClone( + source, target *mongo.Client, + catalog *Catalog, + nsFilter sel.NSFilter, + opts *StartOptions, +) *Clone { return &Clone{ source: source, target: target, catalog: catalog, nsFilter: nsFilter, + options: opts, doneSig: make(chan struct{}), } } @@ -292,7 +300,7 @@ func (c *Clone) run() error { func (c *Clone) doClone(ctx context.Context, namespaces []namespaceInfo) error { cloneLogger := log.Ctx(ctx) - numParallelCollections := config.CloneNumParallelCollections() + numParallelCollections := c.options.CloneParallelism if numParallelCollections < 1 { numParallelCollections = config.DefaultCloneNumParallelCollection } @@ -300,10 +308,10 @@ func (c *Clone) doClone(ctx context.Context, namespaces []namespaceInfo) error { cloneLogger.Debugf("NumParallelCollections: %d", numParallelCollections) copyManager := NewCopyManager(c.source, c.target, CopyManagerOptions{ - NumReadWorkers: config.CloneNumReadWorkers(), - NumInsertWorkers: config.CloneNumInsertWorkers(), - SegmentSizeBytes: config.CloneSegmentSizeBytes(), - ReadBatchSizeBytes: config.CloneReadBatchSizeBytes(), + NumReadWorkers: c.options.CloneReadWorkers, + NumInsertWorkers: c.options.CloneInsertWorkers, + SegmentSizeBytes: c.options.CloneSegmentSizeBytes, + ReadBatchSizeBytes: c.options.CloneReadBatchSizeBytes, }) defer copyManager.Close() diff --git a/pcsm/copy_test.go b/pcsm/copy_test.go index b032dec..ccbd1c9 100644 --- a/pcsm/copy_test.go +++ b/pcsm/copy_test.go @@ -93,7 +93,8 @@ func BenchmarkRead(b *testing.B) { ctx := b.Context() ns := getNamespace() - mc, err := topo.Connect(ctx, getSourceURI()) + cfg := &config.Config{} + mc, err := topo.Connect(ctx, getSourceURI(), cfg) if err != nil { b.Fatal(err) } @@ -147,7 +148,8 @@ func BenchmarkInsert(b *testing.B) { ctx := b.Context() ns := getNamespace() - mc, err := topo.Connect(ctx, getTargetURI()) + cfg := &config.Config{} + mc, err := topo.Connect(ctx, getTargetURI(), cfg) if err != nil { b.Fatal(err) } diff --git a/pcsm/pcsm.go b/pcsm/pcsm.go index b6b91f7..6c0b661 100644 --- a/pcsm/pcsm.go +++ b/pcsm/pcsm.go @@ -167,8 +167,10 @@ func (ml *PCSM) Recover(ctx context.Context, data []byte) error { nsFilter := sel.MakeFilter(cp.NSInclude, cp.NSExclude) catalog := NewCatalog(ml.target) - clone := NewClone(ml.source, ml.target, catalog, nsFilter) - repl := NewRepl(ml.source, ml.target, catalog, nsFilter) + // Use default options for recovery (clone tuning is less relevant when resuming from checkpoint) + defaultOpts := &StartOptions{} + clone := NewClone(ml.source, ml.target, catalog, nsFilter, defaultOpts) + repl := NewRepl(ml.source, ml.target, catalog, nsFilter, false) if cp.Catalog != nil { err = catalog.Recover(cp.Catalog) @@ -334,8 +336,8 @@ func (ml *PCSM) Start(_ context.Context, options *StartOptions) error { ml.nsFilter = sel.MakeFilter(ml.nsInclude, ml.nsExclude) ml.pauseOnInitialSync = options.PauseOnInitialSync ml.catalog = NewCatalog(ml.target) - ml.clone = NewClone(ml.source, ml.target, ml.catalog, ml.nsFilter) - ml.repl = NewRepl(ml.source, ml.target, ml.catalog, ml.nsFilter) + ml.clone = NewClone(ml.source, ml.target, ml.catalog, ml.nsFilter, options) + ml.repl = NewRepl(ml.source, ml.target, ml.catalog, ml.nsFilter, options.UseCollectionBulkWrite) ml.state = StateRunning go ml.run() diff --git a/pcsm/repl.go b/pcsm/repl.go index cd4f84f..07f7bb7 100644 --- a/pcsm/repl.go +++ b/pcsm/repl.go @@ -36,6 +36,8 @@ type Repl struct { nsFilter sel.NSFilter // Namespace filter catalog *Catalog // Catalog for managing collections and indexes + useCollectionBulkWrite bool // Whether to use collection-level bulk write + lastReplicatedOpTime bson.Timestamp lock sync.Mutex @@ -84,14 +86,21 @@ func (rs *ReplStatus) IsPaused() bool { return !rs.PauseTime.IsZero() } -func NewRepl(source, target *mongo.Client, catalog *Catalog, nsFilter sel.NSFilter) *Repl { +// NewRepl creates a new Repl instance. +func NewRepl( + source, target *mongo.Client, + catalog *Catalog, + nsFilter sel.NSFilter, + useCollectionBulkWrite bool, +) *Repl { return &Repl{ - source: source, - target: target, - nsFilter: nsFilter, - catalog: catalog, - pauseC: make(chan struct{}), - doneSig: make(chan struct{}), + source: source, + target: target, + nsFilter: nsFilter, + catalog: catalog, + useCollectionBulkWrite: useCollectionBulkWrite, + pauseC: make(chan struct{}), + doneSig: make(chan struct{}), } } @@ -221,7 +230,7 @@ func (r *Repl) Start(ctx context.Context, startAt bson.Timestamp) error { return errors.Wrap(err, "major version") } - if topo.Support(targetVer).ClientBulkWrite() && !config.UseCollectionBulkWrite() { + if topo.Support(targetVer).ClientBulkWrite() && !r.useCollectionBulkWrite { r.bulkWrite = newClientBulkWrite(config.BulkOpsSize, targetVer.Major() < 8) //nolint:mnd } else { r.bulkWrite = newCollectionBulkWrite(config.BulkOpsSize, targetVer.Major() < 8) //nolint:mnd diff --git a/tests/perf_test.go b/tests/perf_test.go index 0e2aa2e..6350be1 100644 --- a/tests/perf_test.go +++ b/tests/perf_test.go @@ -27,7 +27,8 @@ func BenchmarkInsertOne(b *testing.B) { b.Fatal("no MongoDB URI provided") } - client, err := topo.Connect(b.Context(), mongodbURI) + cfg := &config.Config{} + client, err := topo.Connect(b.Context(), mongodbURI, cfg) if err != nil { b.Fatalf("Failed to connect to MongoDB: %v", err) } @@ -63,7 +64,8 @@ func BenchmarkReplaceOne(b *testing.B) { b.Fatal("no MongoDB URI provided") } - client, err := topo.Connect(b.Context(), mongodbURI) + cfg := &config.Config{} + client, err := topo.Connect(b.Context(), mongodbURI, cfg) if err != nil { b.Fatalf("Failed to connect to MongoDB: %v", err) } @@ -138,14 +140,15 @@ func performIndexTest(b *testing.B, opts performIndexTestOptions) { } ctx := b.Context() + cfg := &config.Config{} - source, err := topo.Connect(ctx, sourceURI) + source, err := topo.Connect(ctx, sourceURI, cfg) if err != nil { b.Fatalf("Failed to connect to MongoDB: %v", err) } defer source.Disconnect(ctx) //nolint:errcheck - target, err := topo.Connect(ctx, targetURI) + target, err := topo.Connect(ctx, targetURI, cfg) if err != nil { b.Fatalf("Failed to connect to MongoDB: %v", err) } diff --git a/topo/connect.go b/topo/connect.go index c140f61..672aac7 100644 --- a/topo/connect.go +++ b/topo/connect.go @@ -25,8 +25,8 @@ type ConnectOptions struct { // Connect establishes a connection to a MongoDB instance using the provided URI. // If the URI is empty, it returns an error. -func Connect(ctx context.Context, uri string) (*mongo.Client, error) { - return ConnectWithOptions(ctx, uri, &ConnectOptions{}) +func Connect(ctx context.Context, uri string, cfg *config.Config) (*mongo.Client, error) { + return ConnectWithOptions(ctx, uri, cfg, &ConnectOptions{}) } // ConnectWithOptions establishes a connection to a MongoDB instance using the provided URI and options. @@ -34,6 +34,7 @@ func Connect(ctx context.Context, uri string) (*mongo.Client, error) { func ConnectWithOptions( ctx context.Context, uri string, + cfg *config.Config, connOpts *ConnectOptions, ) (*mongo.Client, error) { if uri == "" { @@ -58,7 +59,7 @@ func ConnectWithOptions( SetReadPreference(readpref.Primary()). SetReadConcern(readconcern.Majority()). SetWriteConcern(writeconcern.Majority()). - SetTimeout(config.OperationMongoDBCliTimeout()) + SetTimeout(cfg.MongoDB.OperationTimeoutDuration()) if connOpts != nil && connOpts.Compressors != nil { opts.SetCompressors(connOpts.Compressors) diff --git a/validate/validate.go b/validate/validate.go index d7e7f25..f46f744 100644 --- a/validate/validate.go +++ b/validate/validate.go @@ -5,6 +5,7 @@ import ( "reflect" "strings" "sync" + "time" "github.com/go-playground/validator/v10" ) @@ -29,6 +30,19 @@ func registerCustomValidators(v *validator.Validate) { _ = v.RegisterValidation("bytesize", validateByteSize) _ = v.RegisterValidation("bytesizemin", validateByteSizeMin) _ = v.RegisterValidation("bytesizemax", validateByteSizeMax) + _ = v.RegisterValidation("duration", validateDuration) +} + +// validateDuration validates that a string can be parsed as a time.Duration. +func validateDuration(fl validator.FieldLevel) bool { + s := fl.Field().String() + if s == "" { + return true + } + + _, err := time.ParseDuration(s) + + return err == nil } // registerTagNameFunc uses JSON tag names in error messages. From 1ae84b6d4c221ce6a8c0515ad7c549f8346ea99a Mon Sep 17 00:00:00 2001 From: Adnan Date: Wed, 24 Dec 2025 17:03:39 +0100 Subject: [PATCH 15/17] PCSM-219: Use embedded options for Clone and Repl --- main.go | 23 ++++++++++++----------- pcsm/clone.go | 33 ++++++++++++++++++++++++++------- pcsm/pcsm.go | 33 +++++++++++---------------------- pcsm/repl.go | 27 +++++++++++++++++---------- 4 files changed, 66 insertions(+), 50 deletions(-) diff --git a/main.go b/main.go index 8435eee..9c636b1 100644 --- a/main.go +++ b/main.go @@ -712,27 +712,31 @@ func resolveStartOptions(cfg *config.Config, params startRequest) (*pcsm.StartOp PauseOnInitialSync: params.PauseOnInitialSync, IncludeNamespaces: params.IncludeNamespaces, ExcludeNamespaces: params.ExcludeNamespaces, + Repl: pcsm.ReplOptions{ + UseCollectionBulkWrite: cfg.UseCollectionBulkWrite, + }, + Clone: pcsm.CloneOptions{}, } // Clone parallelism: HTTP > CLI > default if params.CloneNumParallelCollections != nil { - options.CloneParallelism = *params.CloneNumParallelCollections + options.Clone.Parallelism = *params.CloneNumParallelCollections } else { - options.CloneParallelism = cfg.Clone.NumParallelCollections + options.Clone.Parallelism = cfg.Clone.NumParallelCollections } // Clone read workers: HTTP > CLI > default if params.CloneNumReadWorkers != nil { - options.CloneReadWorkers = *params.CloneNumReadWorkers + options.Clone.ReadWorkers = *params.CloneNumReadWorkers } else { - options.CloneReadWorkers = cfg.Clone.NumReadWorkers + options.Clone.ReadWorkers = cfg.Clone.NumReadWorkers } // Clone insert workers: HTTP > CLI > default if params.CloneNumInsertWorkers != nil { - options.CloneInsertWorkers = *params.CloneNumInsertWorkers + options.Clone.InsertWorkers = *params.CloneNumInsertWorkers } else { - options.CloneInsertWorkers = cfg.Clone.NumInsertWorkers + options.Clone.InsertWorkers = cfg.Clone.NumInsertWorkers } // Clone segment size: HTTP > CLI > default @@ -740,17 +744,14 @@ func resolveStartOptions(cfg *config.Config, params startRequest) (*pcsm.StartOp if err != nil { return nil, err } - options.CloneSegmentSizeBytes = segmentSize + options.Clone.SegmentSizeBytes = segmentSize // Clone read batch size: HTTP > CLI > default batchSize, err := resolveCloneReadBatchSize(cfg, params.CloneReadBatchSize) if err != nil { return nil, err } - options.CloneReadBatchSizeBytes = batchSize - - // UseCollectionBulkWrite: internal only, always from config (CLI + env var via Viper) - options.UseCollectionBulkWrite = cfg.UseCollectionBulkWrite + options.Clone.ReadBatchSizeBytes = batchSize return options, nil } diff --git a/pcsm/clone.go b/pcsm/clone.go index d3cc25c..2f08aea 100644 --- a/pcsm/clone.go +++ b/pcsm/clone.go @@ -22,13 +22,32 @@ import ( "github.com/percona/percona-clustersync-mongodb/topo" ) +// CloneOptions configures the clone behavior. +type CloneOptions struct { + // Parallelism is the number of collections to clone in parallel. + // Default: 2 (config.DefaultCloneNumParallelCollection) + Parallelism int + // ReadWorkers is the number of read workers during clone. + // Default: auto (0 = runtime.NumCPU()/4) + ReadWorkers int + // InsertWorkers is the number of insert workers during clone. + // Default: auto (0 = runtime.NumCPU()*2) + InsertWorkers int + // SegmentSizeBytes is the segment size for clone operations in bytes. + // Default: auto (0 = calculated per collection) + SegmentSizeBytes int64 + // ReadBatchSizeBytes is the read batch size during clone in bytes. + // Default: ~47.5MB (config.DefaultCloneReadBatchSizeBytes) + ReadBatchSizeBytes int32 +} + // Clone handles the cloning of data from a source MongoDB to a target MongoDB. type Clone struct { source *mongo.Client // Source MongoDB client target *mongo.Client // Target MongoDB client catalog *Catalog // Catalog for managing collections and indexes nsFilter sel.NSFilter // Namespace filter - options *StartOptions // Clone options from StartOptions + options *CloneOptions // Clone options lock sync.Mutex err error // Error encountered during the cloning process @@ -80,7 +99,7 @@ func NewClone( source, target *mongo.Client, catalog *Catalog, nsFilter sel.NSFilter, - opts *StartOptions, + opts *CloneOptions, ) *Clone { return &Clone{ source: source, @@ -300,7 +319,7 @@ func (c *Clone) run() error { func (c *Clone) doClone(ctx context.Context, namespaces []namespaceInfo) error { cloneLogger := log.Ctx(ctx) - numParallelCollections := c.options.CloneParallelism + numParallelCollections := c.options.Parallelism if numParallelCollections < 1 { numParallelCollections = config.DefaultCloneNumParallelCollection } @@ -308,10 +327,10 @@ func (c *Clone) doClone(ctx context.Context, namespaces []namespaceInfo) error { cloneLogger.Debugf("NumParallelCollections: %d", numParallelCollections) copyManager := NewCopyManager(c.source, c.target, CopyManagerOptions{ - NumReadWorkers: c.options.CloneReadWorkers, - NumInsertWorkers: c.options.CloneInsertWorkers, - SegmentSizeBytes: c.options.CloneSegmentSizeBytes, - ReadBatchSizeBytes: c.options.CloneReadBatchSizeBytes, + NumReadWorkers: c.options.ReadWorkers, + NumInsertWorkers: c.options.InsertWorkers, + SegmentSizeBytes: c.options.SegmentSizeBytes, + ReadBatchSizeBytes: c.options.ReadBatchSizeBytes, }) defer copyManager.Close() diff --git a/pcsm/pcsm.go b/pcsm/pcsm.go index 6c0b661..16bb3ad 100644 --- a/pcsm/pcsm.go +++ b/pcsm/pcsm.go @@ -168,9 +168,8 @@ func (ml *PCSM) Recover(ctx context.Context, data []byte) error { nsFilter := sel.MakeFilter(cp.NSInclude, cp.NSExclude) catalog := NewCatalog(ml.target) // Use default options for recovery (clone tuning is less relevant when resuming from checkpoint) - defaultOpts := &StartOptions{} - clone := NewClone(ml.source, ml.target, catalog, nsFilter, defaultOpts) - repl := NewRepl(ml.source, ml.target, catalog, nsFilter, false) + clone := NewClone(ml.source, ml.target, catalog, nsFilter, &CloneOptions{}) + repl := NewRepl(ml.source, ml.target, catalog, nsFilter, &ReplOptions{}) if cp.Catalog != nil { err = catalog.Recover(cp.Catalog) @@ -285,27 +284,17 @@ func (ml *PCSM) resetError() { // StartOptions represents the options for starting the PCSM. type StartOptions struct { - // PauseOnInitialSync indicates whether to finalize after the initial sync. + // PauseOnInitialSync indicates whether to pause after the initial sync completes. PauseOnInitialSync bool - // IncludeNamespaces are the namespaces to include. + // IncludeNamespaces are the namespaces to include in replication. IncludeNamespaces []string - // ExcludeNamespaces are the namespaces to exclude. + // ExcludeNamespaces are the namespaces to exclude from replication. ExcludeNamespaces []string - // Clone tuning options - // CloneParallelism is the number of collections to clone in parallel. - CloneParallelism int - // CloneReadWorkers is the number of read workers during clone. - CloneReadWorkers int - // CloneInsertWorkers is the number of insert workers during clone. - CloneInsertWorkers int - // CloneSegmentSizeBytes is the segment size for clone operations in bytes. - CloneSegmentSizeBytes int64 - // CloneReadBatchSizeBytes is the read batch size during clone in bytes. - CloneReadBatchSizeBytes int32 - - // UseCollectionBulkWrite indicates whether to use collection-level bulk write. - UseCollectionBulkWrite bool + // Clone contains clone tuning options. + Clone CloneOptions + // Repl contains replication behavior options. + Repl ReplOptions } // Start starts the replication process with the given options. @@ -336,8 +325,8 @@ func (ml *PCSM) Start(_ context.Context, options *StartOptions) error { ml.nsFilter = sel.MakeFilter(ml.nsInclude, ml.nsExclude) ml.pauseOnInitialSync = options.PauseOnInitialSync ml.catalog = NewCatalog(ml.target) - ml.clone = NewClone(ml.source, ml.target, ml.catalog, ml.nsFilter, options) - ml.repl = NewRepl(ml.source, ml.target, ml.catalog, ml.nsFilter, options.UseCollectionBulkWrite) + ml.clone = NewClone(ml.source, ml.target, ml.catalog, ml.nsFilter, &options.Clone) + ml.repl = NewRepl(ml.source, ml.target, ml.catalog, ml.nsFilter, &options.Repl) ml.state = StateRunning go ml.run() diff --git a/pcsm/repl.go b/pcsm/repl.go index 07f7bb7..67f4a14 100644 --- a/pcsm/repl.go +++ b/pcsm/repl.go @@ -28,6 +28,13 @@ var ( const advanceTimePseudoEvent = "@tick" +// ReplOptions configures the replication behavior. +type ReplOptions struct { + // UseCollectionBulkWrite indicates whether to use collection-level bulk write + // instead of client bulk write. Default: false (use client bulk write). + UseCollectionBulkWrite bool +} + // Repl handles replication from a source MongoDB to a target MongoDB. type Repl struct { source *mongo.Client // Source MongoDB client @@ -36,7 +43,7 @@ type Repl struct { nsFilter sel.NSFilter // Namespace filter catalog *Catalog // Catalog for managing collections and indexes - useCollectionBulkWrite bool // Whether to use collection-level bulk write + options *ReplOptions // Replication options lastReplicatedOpTime bson.Timestamp @@ -91,16 +98,16 @@ func NewRepl( source, target *mongo.Client, catalog *Catalog, nsFilter sel.NSFilter, - useCollectionBulkWrite bool, + opts *ReplOptions, ) *Repl { return &Repl{ - source: source, - target: target, - nsFilter: nsFilter, - catalog: catalog, - useCollectionBulkWrite: useCollectionBulkWrite, - pauseC: make(chan struct{}), - doneSig: make(chan struct{}), + source: source, + target: target, + nsFilter: nsFilter, + catalog: catalog, + options: opts, + pauseC: make(chan struct{}), + doneSig: make(chan struct{}), } } @@ -230,7 +237,7 @@ func (r *Repl) Start(ctx context.Context, startAt bson.Timestamp) error { return errors.Wrap(err, "major version") } - if topo.Support(targetVer).ClientBulkWrite() && !r.useCollectionBulkWrite { + if topo.Support(targetVer).ClientBulkWrite() && !r.options.UseCollectionBulkWrite { r.bulkWrite = newClientBulkWrite(config.BulkOpsSize, targetVer.Major() < 8) //nolint:mnd } else { r.bulkWrite = newCollectionBulkWrite(config.BulkOpsSize, targetVer.Major() < 8) //nolint:mnd From 2b30ea261cbe83f38bd189505786dde99c028f63 Mon Sep 17 00:00:00 2001 From: Adnan Date: Wed, 24 Dec 2025 17:16:56 +0100 Subject: [PATCH 16/17] PCSM-219: Undo README changes --- README.md | 87 +++++++++---------------------------------------------- 1 file changed, 13 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index 4b42d95..7430426 100644 --- a/README.md +++ b/README.md @@ -145,15 +145,12 @@ curl http://localhost:2242/status When starting the PCSM server, you can use the following options: -| Option | CLI Flag | Default | Description | -|----------------------------------|------------------------------------|---------|--------------------------------------| -| Port | `--port` | 2242 | Port on which the server listens | -| Source URI | `--source` | - | MongoDB connection string for source | -| Target URI | `--target` | - | MongoDB connection string for target | -| Log Level | `--log-level` | info | Log level (trace/debug/info/warn/error/fatal/panic) | -| Log JSON | `--log-json` | false | Output log in JSON format | -| No Color | `--no-color` | false | Disable log ASCII color | -| MongoDB Operation Timeout | `--mongodb-cli-operation-timeout` | 5m | Timeout for MongoDB operations | +- `--port`: The port on which the server will listen (default: 2242) +- `--source`: The MongoDB connection string for the source cluster +- `--target`: The MongoDB connection string for the target cluster +- `--log-level`: The log level (default: "info") +- `--log-json`: Output log in JSON format with disabled color +- `--no-color`: Disable log ASCI color Example: @@ -163,52 +160,14 @@ bin/pcsm \ --target \ --port 2242 \ --log-level debug \ - --log-json \ - --mongodb-cli-operation-timeout 10m + --log-json ``` ## Environment Variables -The following environment variables are supported: - -| Option | Env Var | Default | Description | -|----------------------------|--------------------------------------|---------|--------------------------------------| -| Source URI | `PCSM_SOURCE_URI` | - | MongoDB connection string for source | -| Target URI | `PCSM_TARGET_URI` | - | MongoDB connection string for target | -| Port | `PCSM_PORT` | 2242 | Port on which the server listens | -| Log Level | `PCSM_LOG_LEVEL` | info | Log level | -| Log JSON | `PCSM_LOG_JSON` | false | Output log in JSON format | -| No Color | `PCSM_NO_COLOR` | false | Disable log ASCII color | -| MongoDB Operation Timeout | `PCSM_MONGODB_CLI_OPERATION_TIMEOUT` | 5m | Timeout for MongoDB operations | -| Use Collection Bulk Write | `PCSM_USE_COLLECTION_BULK_WRITE` | false | Use collection-level bulk write (internal) | - -> **Note**: Clone tuning options (see below) are intentionally NOT supported via environment variables. They are configurable via CLI flags and HTTP request parameters only. - -## Clone Tuning Options - -Advanced tuning options for the clone process. These are available via CLI flags -and HTTP request parameters, but NOT via environment variables. - -| CLI Flag | HTTP Parameter | Default | Range | Description | -|------------------------------------|-------------------------------|----------|-------------|------------------------------------| -| `--clone-num-parallel-collections` | `cloneNumParallelCollections` | 2 | 0-100 | Collections to clone in parallel | -| `--clone-num-read-workers` | `cloneNumReadWorkers` | auto (0) | 0-1000 | Read workers during clone | -| `--clone-num-insert-workers` | `cloneNumInsertWorkers` | auto (0) | 0-1000 | Insert workers during clone | -| `--clone-segment-size` | `cloneSegmentSize` | auto | ~475MB-64GB | Segment size for parallel cloning | -| `--clone-read-batch-size` | `cloneReadBatchSize` | ~47.5MB | 16MiB-2GiB | Read cursor batch size | - -> **Note**: These CLI flags are hidden from `--help` output. They are intended for advanced tuning only. -> Setting a value to 0 or empty string uses the automatic/default behavior. - -Example CLI usage: - -```sh -bin/pcsm \ - --source \ - --target \ - --clone-num-parallel-collections 8 \ - --clone-num-read-workers 16 -``` +- `PCSM_SOURCE_URI`: MongoDB connection string for the source cluster. +- `PCSM_TARGET_URI`: MongoDB connection string for the target cluster. +- `PCSM_MONGODB_CLI_OPERATION_TIMEOUT`: Timeout for MongoDB client operations; accepts Go durations like `30s`, `2m`, `1h` (default: `5m`). ## Log JSON Fields @@ -247,19 +206,10 @@ Starts the replication process. #### Request Body -| Parameter | Type | Description | -|-------------------------------|----------|-------------------------------------------------| -| `includeNamespaces` | string[] | Namespaces to include in replication | -| `excludeNamespaces` | string[] | Namespaces to exclude from replication | -| `cloneNumParallelCollections` | int | Collections to clone in parallel (0-100) | -| `cloneNumReadWorkers` | int | Read workers during clone (0-1000) | -| `cloneNumInsertWorkers` | int | Insert workers during clone (0-1000) | -| `cloneSegmentSize` | string | Segment size (e.g., "500MB", "1GiB") | -| `cloneReadBatchSize` | string | Read batch size (e.g., "32MiB") | - -> **Note**: HTTP request values take precedence over CLI flag values for clone tuning options. +- `includeNamespaces` (optional): List of namespaces to include in the replication. +- `excludeNamespaces` (optional): List of namespaces to exclude from the replication. -Example (basic): +Example: ```json { @@ -268,17 +218,6 @@ Example (basic): } ``` -Example (with clone tuning): - -```json -{ - "includeNamespaces": ["mydb.*"], - "cloneNumParallelCollections": 8, - "cloneNumReadWorkers": 16, - "cloneSegmentSize": "500MB" -} -``` - #### Response - `ok`: Boolean indicating if the operation was successful. From 2a0e34c5c4305d4714f31cb14599c9b9cfd7ce5f Mon Sep 17 00:00:00 2001 From: Adnan Date: Fri, 26 Dec 2025 14:35:22 +0100 Subject: [PATCH 17/17] PCSM-219: Undoing using the validator --- config/config.go | 6 ---- config/schema.go | 16 ++++----- go.mod | 7 +--- go.sum | 12 ------- main.go | 23 +++---------- validate/bytesize.go | 74 ----------------------------------------- validate/errors.go | 78 -------------------------------------------- validate/validate.go | 63 ----------------------------------- 8 files changed, 14 insertions(+), 265 deletions(-) delete mode 100644 validate/bytesize.go delete mode 100644 validate/errors.go delete mode 100644 validate/validate.go diff --git a/config/config.go b/config/config.go index c1539e3..2a2814a 100644 --- a/config/config.go +++ b/config/config.go @@ -10,7 +10,6 @@ import ( "github.com/spf13/viper" "github.com/percona/percona-clustersync-mongodb/errors" - "github.com/percona/percona-clustersync-mongodb/validate" ) // Load initializes Viper and returns a validated Config. @@ -35,11 +34,6 @@ func Load(cmd *cobra.Command) (*Config, error) { return nil, errors.Wrap(err, "unmarshal config") } - err = validate.Struct(&cfg) - if err != nil { - return nil, errors.Wrap(err, "validate config") - } - return &cfg, nil } diff --git a/config/schema.go b/config/schema.go index 06401e0..864385d 100644 --- a/config/schema.go +++ b/config/schema.go @@ -10,7 +10,7 @@ import ( // Config holds all PCSM configuration. type Config struct { // Connection - Port int `mapstructure:"port" validate:"omitempty,gte=1024,lte=65535"` + Port int `mapstructure:"port"` Source string `mapstructure:"source"` Target string `mapstructure:"target"` @@ -34,14 +34,14 @@ type Config struct { // LogConfig holds logging configuration. type LogConfig struct { - Level string `mapstructure:"log-level" validate:"omitempty,oneof=trace debug info warn error fatal panic"` + Level string `mapstructure:"log-level"` JSON bool `mapstructure:"log-json"` NoColor bool `mapstructure:"no-color"` } // MongoDBConfig holds MongoDB client configuration. type MongoDBConfig struct { - OperationTimeout string `mapstructure:"mongodb-cli-operation-timeout" validate:"omitempty,duration"` + OperationTimeout string `mapstructure:"mongodb-cli-operation-timeout"` } // OperationTimeoutDuration returns the parsed timeout or default. @@ -58,11 +58,11 @@ func (m *MongoDBConfig) OperationTimeoutDuration() time.Duration { // CloneConfig holds clone tuning configuration. type CloneConfig struct { - NumParallelCollections int `mapstructure:"clone-num-parallel-collections" validate:"omitempty,gte=0,lte=100"` - NumReadWorkers int `mapstructure:"clone-num-read-workers" validate:"omitempty,gte=0,lte=1000"` - NumInsertWorkers int `mapstructure:"clone-num-insert-workers" validate:"omitempty,gte=0,lte=1000"` - SegmentSize string `mapstructure:"clone-segment-size" validate:"omitempty,bytesize"` - ReadBatchSize string `mapstructure:"clone-read-batch-size" validate:"omitempty,bytesize"` + NumParallelCollections int `mapstructure:"clone-num-parallel-collections"` + NumReadWorkers int `mapstructure:"clone-num-read-workers"` + NumInsertWorkers int `mapstructure:"clone-num-insert-workers"` + SegmentSize string `mapstructure:"clone-segment-size"` + ReadBatchSize string `mapstructure:"clone-read-batch-size"` } // SegmentSizeBytes parses and returns the segment size in bytes. diff --git a/go.mod b/go.mod index 6f29ab1..0929c43 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,9 @@ go 1.25.0 require ( github.com/dustin/go-humanize v1.0.1 - github.com/go-playground/validator/v10 v10.30.0 github.com/prometheus/client_golang v1.22.0 github.com/rs/zerolog v1.34.0 github.com/spf13/cobra v1.9.1 - github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 go.mongodb.org/mongo-driver/v2 v2.2.1 @@ -20,14 +18,10 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.12 // indirect - github.com/go-playground/locales v0.14.1 // indirect - github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/compress v1.18.0 // indirect - github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -40,6 +34,7 @@ require ( github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect diff --git a/go.sum b/go.sum index c101f33..e2b8ab4 100644 --- a/go.sum +++ b/go.sum @@ -12,16 +12,6 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= -github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= -github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= -github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= -github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= -github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= -github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.30.0 h1:5YBPNs273uzsZJD1I8uiB4Aqg9sN6sMDVX3s6LxmhWU= -github.com/go-playground/validator/v10 v10.30.0/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -39,8 +29,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= -github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= diff --git a/main.go b/main.go index 9c636b1..7dc18b5 100644 --- a/main.go +++ b/main.go @@ -30,7 +30,6 @@ import ( "github.com/percona/percona-clustersync-mongodb/pcsm" "github.com/percona/percona-clustersync-mongodb/topo" "github.com/percona/percona-clustersync-mongodb/util" - "github.com/percona/percona-clustersync-mongodb/validate" ) // Constants for server configuration. @@ -851,13 +850,6 @@ func (s *server) handleStart(w http.ResponseWriter, r *http.Request) { } } - // Validate request - if err := params.Validate(); err != nil { - writeResponse(w, startResponse{Err: err.Error()}) - - return - } - options, err := resolveStartOptions(s.cfg, params) if err != nil { writeResponse(w, startResponse{Err: err.Error()}) @@ -1046,24 +1038,19 @@ type startRequest struct { // Clone tuning options (pointer types to distinguish "not set" from zero value) // CloneNumParallelCollections is the number of collections to clone in parallel. - CloneNumParallelCollections *int `json:"cloneNumParallelCollections,omitempty" validate:"omitempty,gte=0,lte=100"` + CloneNumParallelCollections *int `json:"cloneNumParallelCollections,omitempty"` // CloneNumReadWorkers is the number of read workers during clone. - CloneNumReadWorkers *int `json:"cloneNumReadWorkers,omitempty" validate:"omitempty,gte=0,lte=1000"` + CloneNumReadWorkers *int `json:"cloneNumReadWorkers,omitempty"` // CloneNumInsertWorkers is the number of insert workers during clone. - CloneNumInsertWorkers *int `json:"cloneNumInsertWorkers,omitempty" validate:"omitempty,gte=0,lte=1000"` + CloneNumInsertWorkers *int `json:"cloneNumInsertWorkers,omitempty"` // CloneSegmentSize is the segment size for clone operations (e.g., "100MB", "1GiB"). - CloneSegmentSize *string `json:"cloneSegmentSize,omitempty" validate:"omitempty,bytesize"` + CloneSegmentSize *string `json:"cloneSegmentSize,omitempty"` // CloneReadBatchSize is the read batch size during clone (e.g., "16MiB"). - CloneReadBatchSize *string `json:"cloneReadBatchSize,omitempty" validate:"omitempty,bytesize"` + CloneReadBatchSize *string `json:"cloneReadBatchSize,omitempty"` // NOTE: UseCollectionBulkWrite intentionally NOT exposed via HTTP (internal only) } -// Validate validates the startRequest. -func (r *startRequest) Validate() error { - return validate.Struct(r) -} - // startResponse represents the response body for the /start endpoint. type startResponse struct { // Ok indicates if the operation was successful. diff --git a/validate/bytesize.go b/validate/bytesize.go deleted file mode 100644 index 54f59a3..0000000 --- a/validate/bytesize.go +++ /dev/null @@ -1,74 +0,0 @@ -package validate - -import ( - "reflect" - - "github.com/dustin/go-humanize" - "github.com/go-playground/validator/v10" -) - -// validateByteSize checks if a string can be parsed as a byte size. -func validateByteSize(fl validator.FieldLevel) bool { - s := getStringValue(fl.Field()) - if s == "" || s == "0" { - return true // empty/zero = use default - } - - _, err := humanize.ParseBytes(s) - - return err == nil -} - -// validateByteSizeMin validates minimum byte size. -// Tag usage: bytesizemin=16MB -func validateByteSizeMin(fl validator.FieldLevel) bool { - s := getStringValue(fl.Field()) - if s == "" || s == "0" { - return true // empty/zero bypasses validation (use default) - } - - bytes, err := humanize.ParseBytes(s) - if err != nil { - return false - } - - minBytes, err := humanize.ParseBytes(fl.Param()) - if err != nil { - return false - } - - return bytes >= minBytes -} - -// validateByteSizeMax validates maximum byte size. -// Tag usage: bytesizemax=64GiB -func validateByteSizeMax(fl validator.FieldLevel) bool { - s := getStringValue(fl.Field()) - if s == "" || s == "0" { - return true - } - - bytes, err := humanize.ParseBytes(s) - if err != nil { - return false - } - - maxBytes, err := humanize.ParseBytes(fl.Param()) - if err != nil { - return false - } - - return bytes <= maxBytes -} - -func getStringValue(field reflect.Value) string { - if field.Kind() == reflect.Ptr { - if field.IsNil() { - return "" - } - - return field.Elem().String() - } - - return field.String() -} diff --git a/validate/errors.go b/validate/errors.go deleted file mode 100644 index 5c58d24..0000000 --- a/validate/errors.go +++ /dev/null @@ -1,78 +0,0 @@ -package validate - -import ( - "fmt" - "strings" - - "github.com/go-playground/validator/v10" -) - -// ValidationError represents a single validation error. -type ValidationError struct { - Field string `json:"field"` - Message string `json:"message"` -} - -func (e ValidationError) Error() string { - return fmt.Sprintf("%s: %s", e.Field, e.Message) -} - -// ValidationErrors is a collection of validation errors. -type ValidationErrors []ValidationError - -func (e ValidationErrors) Error() string { - var sb strings.Builder - for i, err := range e { - if i > 0 { - sb.WriteString("; ") - } - sb.WriteString(err.Error()) - } - - return sb.String() -} - -// TranslateErrors converts validator.ValidationErrors to user-friendly messages. -func TranslateErrors(err error) error { - if err == nil { - return nil - } - - validationErrors, ok := err.(validator.ValidationErrors) //nolint:errorlint - if !ok { - return err - } - - var errs ValidationErrors - for _, e := range validationErrors { - errs = append(errs, ValidationError{ - Field: e.Field(), - Message: translateFieldError(e), - }) - } - - return errs -} - -func translateFieldError(e validator.FieldError) string { - switch e.Tag() { - case "required": - return "is required" - case "gte": - return fmt.Sprintf("must be at least %s", e.Param()) - case "lte": - return fmt.Sprintf("must be at most %s", e.Param()) - case "min": - return fmt.Sprintf("must be at least %s", e.Param()) - case "max": - return fmt.Sprintf("must be at most %s", e.Param()) - case "bytesize": - return "must be a valid byte size (e.g., '100MB', '1GiB')" - case "bytesizemin": - return fmt.Sprintf("must be at least %s", e.Param()) - case "bytesizemax": - return fmt.Sprintf("must be at most %s", e.Param()) - default: - return fmt.Sprintf("failed %s validation", e.Tag()) - } -} diff --git a/validate/validate.go b/validate/validate.go deleted file mode 100644 index f46f744..0000000 --- a/validate/validate.go +++ /dev/null @@ -1,63 +0,0 @@ -// Package validate provides request validation using go-playground/validator. -package validate - -import ( - "reflect" - "strings" - "sync" - "time" - - "github.com/go-playground/validator/v10" -) - -var ( - instance *validator.Validate //nolint:gochecknoglobals - once sync.Once //nolint:gochecknoglobals -) - -// Validator returns the singleton validator instance. -func Validator() *validator.Validate { - once.Do(func() { - instance = validator.New(validator.WithRequiredStructEnabled()) - registerCustomValidators(instance) - registerTagNameFunc(instance) - }) - - return instance -} - -func registerCustomValidators(v *validator.Validate) { - _ = v.RegisterValidation("bytesize", validateByteSize) - _ = v.RegisterValidation("bytesizemin", validateByteSizeMin) - _ = v.RegisterValidation("bytesizemax", validateByteSizeMax) - _ = v.RegisterValidation("duration", validateDuration) -} - -// validateDuration validates that a string can be parsed as a time.Duration. -func validateDuration(fl validator.FieldLevel) bool { - s := fl.Field().String() - if s == "" { - return true - } - - _, err := time.ParseDuration(s) - - return err == nil -} - -// registerTagNameFunc uses JSON tag names in error messages. -func registerTagNameFunc(v *validator.Validate) { - v.RegisterTagNameFunc(func(fld reflect.StructField) string { - name := strings.SplitN(fld.Tag.Get("json"), ",", 2)[0] - if name == "-" { - return fld.Name - } - - return name - }) -} - -// Struct validates a struct using the singleton validator. -func Struct(s any) error { - return TranslateErrors(Validator().Struct(s)) -}