diff --git a/config/config.go b/config/config.go index 430fac16a..b2cadac65 100644 --- a/config/config.go +++ b/config/config.go @@ -1711,11 +1711,11 @@ func InitServer(ctx context.Context, currentServers server_structs.ServerType) e } if param.Cache_LowWatermark.IsSet() || param.Cache_HighWaterMark.IsSet() { - lowWm, lwmIsAbs, err := utils.ValidateWatermark(param.Cache_LowWatermark.GetName(), false) + lowWm, lwmIsAbs, err := utils.ValidateWatermark(param.Cache_LowWatermark, false) if err != nil { return err } - highWm, hwmIsAbs, err := utils.ValidateWatermark(param.Cache_HighWaterMark.GetName(), false) + highWm, hwmIsAbs, err := utils.ValidateWatermark(param.Cache_HighWaterMark, false) if err != nil { return err } @@ -1741,13 +1741,13 @@ func InitServer(ctx context.Context, currentServers server_structs.ServerType) e var base, nominal, max float64 var err error // Watermark validation will error if these parameters are not absolute, so we can ignore that output - if base, _, err = utils.ValidateWatermark(param.Cache_FilesBaseSize.GetName(), true); err != nil { + if base, _, err = utils.ValidateWatermark(param.Cache_FilesBaseSize, true); err != nil { return err } - if nominal, _, err = utils.ValidateWatermark(param.Cache_FilesNominalSize.GetName(), true); err != nil { + if nominal, _, err = utils.ValidateWatermark(param.Cache_FilesNominalSize, true); err != nil { return err } - if max, _, err = utils.ValidateWatermark(param.Cache_FilesMaxSize.GetName(), true); err != nil { + if max, _, err = utils.ValidateWatermark(param.Cache_FilesMaxSize, true); err != nil { return err } if base >= nominal || nominal >= max { diff --git a/config/resources/defaults.yaml b/config/resources/defaults.yaml index e73a37a31..0a715e62a 100644 --- a/config/resources/defaults.yaml +++ b/config/resources/defaults.yaml @@ -86,6 +86,7 @@ Origin: EnableMacaroons: false EnableVoms: true SelfTestInterval: 15s + StorageType: "posix" Registry: InstitutionsUrlReloadMinutes: 15m RequireCacheApproval: false @@ -109,6 +110,7 @@ Shoveler: PortLower: 9930 PortHigher: 9999 AMQPExchange: shoveled-xrd + Topic: "xrootd.shoveler" Xrootd: MaxStartupWait: "10s" Mount: "" diff --git a/director/maxmind.go b/director/maxmind.go index 8f1e0c6af..aa02e9098 100644 --- a/director/maxmind.go +++ b/director/maxmind.go @@ -36,7 +36,6 @@ import ( "github.com/oschwald/geoip2-golang/v2" "github.com/pkg/errors" log "github.com/sirupsen/logrus" - "github.com/spf13/viper" "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" @@ -76,17 +75,17 @@ func downloadDB(localFile string) error { var licenseKey string keyFile := param.Director_MaxMindKeyFile.GetString() - keyFromEnv := viper.GetString("MAXMINDKEY") + keyFromParam := param.Director_MaxMindKey.GetString() if keyFile != "" { contents, err := os.ReadFile(keyFile) if err != nil { return err } licenseKey = strings.TrimSpace(string(contents)) - } else if keyFromEnv != "" { - licenseKey = keyFromEnv + } else if keyFromParam != "" { + licenseKey = keyFromParam } else { - return errors.New("A MaxMind key file must be specified in the config (Director.MaxMindKeyFile), in the environment (PELICAN_DIRECTOR_MAXMINDKEYFILE), or the key must be provided via the environment variable PELICAN_MAXMINDKEY)") + return errors.New("A MaxMind key file must be specified in the config (Director.MaxMindKeyFile), in the environment (PELICAN_DIRECTOR_MAXMINDKEYFILE), or the key must be provided via the environment variable PELICAN_DIRECTOR_MAXMINDKEY)") } url := fmt.Sprintf(maxMindURL, licenseKey) diff --git a/docs/parameters.yaml b/docs/parameters.yaml index e5809a2e6..e2b72298e 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -1981,6 +1981,15 @@ type: filename default: none components: ["director"] --- +name: Director.MaxMindKey +description: |+ + A MaxMind API key provided directly as a string. This is an alternative to providing the key via a file + (Director.MaxMindKeyFile). If both are set, the file takes precedence. This parameter can be set via the + environment variable PELICAN_DIRECTOR_MAXMINDKEY. +type: string +default: none +components: ["director"] +--- name: Director.GeoIPLocation description: |+ A filepath to the intended location of the MaxMind GeoLite City database. This option can be used either to load diff --git a/lotman/lotman_linux.go b/lotman/lotman_linux.go index a235e37f3..33244be95 100644 --- a/lotman/lotman_linux.go +++ b/lotman/lotman_linux.go @@ -42,7 +42,6 @@ import ( "github.com/mitchellh/mapstructure" "github.com/pkg/errors" log "github.com/sirupsen/logrus" - "github.com/spf13/viper" "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/param" @@ -483,7 +482,7 @@ func GetPolicyMap() (map[string]PurgePolicy, error) { policyMap := make(map[string]PurgePolicy) var policies []PurgePolicy // Use custom decoder hook to validate fields. This validates all the way down to the bottom of the lot object. - if err := viper.UnmarshalKey(param.Lotman_PolicyDefinitions.GetName(), &policies, viper.DecodeHook(validateFieldsHook())); err != nil { + if err := param.Lotman_PolicyDefinitions.UnmarshalWithHook(&policies, validateFieldsHook()); err != nil { return policyMap, errors.Wrap(err, "error unmarshalling Lotman policy definitions") } diff --git a/metrics/shoveler.go b/metrics/shoveler.go index af93f3303..0195fcdb3 100644 --- a/metrics/shoveler.go +++ b/metrics/shoveler.go @@ -32,7 +32,6 @@ import ( shoveler "github.com/opensciencegrid/xrootd-monitoring-shoveler" "github.com/pkg/errors" log "github.com/sirupsen/logrus" - "github.com/spf13/viper" "golang.org/x/sync/errgroup" "github.com/pelicanplatform/pelican/config" @@ -88,8 +87,6 @@ func configShoveler(c *shoveler.Config) error { return fmt.Errorf("Token content is empty. Reading from Shoveler.AMQPTokenLocation at %s", c.AmqpToken) } } else { // Stomp - viper.SetDefault("Shoveler.Topic", "xrootd.shoveler") - c.StompUser = param.Shoveler_StompUsername.GetString() log.Debugln("STOMP User:", c.StompUser) c.StompPassword = param.Shoveler_StompPassword.GetString() diff --git a/param/parameters.go b/param/parameters.go index b90e12b8a..89f851ebc 100644 --- a/param/parameters.go +++ b/param/parameters.go @@ -155,6 +155,7 @@ var runtimeConfigurableMap = map[string]bool{ "Director.FilterCachesInErrorState": false, "Director.FilteredServers": false, "Director.GeoIPLocation": false, + "Director.MaxMindKey": false, "Director.MaxMindKeyFile": false, "Director.MaxStatResponse": false, "Director.MinStatResponse": false, @@ -469,6 +470,8 @@ func (sP StringParam) GetString() string { return config.Director.DefaultResponse case "Director.GeoIPLocation": return config.Director.GeoIPLocation + case "Director.MaxMindKey": + return config.Director.MaxMindKey case "Director.MaxMindKeyFile": return config.Director.MaxMindKeyFile case "Director.SupportContactEmail": @@ -1159,6 +1162,10 @@ func (oP ObjectParam) Unmarshal(rawVal any) error { return viper.UnmarshalKey(oP.name, rawVal) } +func (oP ObjectParam) UnmarshalWithHook(rawVal any, decodeHook any) error { + return viper.UnmarshalKey(oP.name, rawVal, viper.DecodeHook(decodeHook)) +} + func (oP ObjectParam) GetName() string { return oP.name } @@ -1249,6 +1256,7 @@ var allParameterNames = []string{ "Director.FilterCachesInErrorState", "Director.FilteredServers", "Director.GeoIPLocation", + "Director.MaxMindKey", "Director.MaxMindKeyFile", "Director.MaxStatResponse", "Director.MinStatResponse", @@ -1527,6 +1535,7 @@ var ( Director_DbLocation = StringParam{"Director.DbLocation"} Director_DefaultResponse = StringParam{"Director.DefaultResponse"} Director_GeoIPLocation = StringParam{"Director.GeoIPLocation"} + Director_MaxMindKey = StringParam{"Director.MaxMindKey"} Director_MaxMindKeyFile = StringParam{"Director.MaxMindKeyFile"} Director_SupportContactEmail = StringParam{"Director.SupportContactEmail"} Director_SupportContactUrl = StringParam{"Director.SupportContactUrl"} diff --git a/param/parameters_struct.go b/param/parameters_struct.go index 357458875..a2a357a0d 100644 --- a/param/parameters_struct.go +++ b/param/parameters_struct.go @@ -104,6 +104,7 @@ type Config struct { FilterCachesInErrorState bool `mapstructure:"filtercachesinerrorstate" yaml:"FilterCachesInErrorState"` FilteredServers []string `mapstructure:"filteredservers" yaml:"FilteredServers"` GeoIPLocation string `mapstructure:"geoiplocation" yaml:"GeoIPLocation"` + MaxMindKey string `mapstructure:"maxmindkey" yaml:"MaxMindKey"` MaxMindKeyFile string `mapstructure:"maxmindkeyfile" yaml:"MaxMindKeyFile"` MaxStatResponse int `mapstructure:"maxstatresponse" yaml:"MaxStatResponse"` MinStatResponse int `mapstructure:"minstatresponse" yaml:"MinStatResponse"` @@ -484,6 +485,7 @@ type configWithType struct { FilterCachesInErrorState struct { Type string; Value bool } FilteredServers struct { Type string; Value []string } GeoIPLocation struct { Type string; Value string } + MaxMindKey struct { Type string; Value string } MaxMindKeyFile struct { Type string; Value string } MaxStatResponse struct { Type string; Value int } MinStatResponse struct { Type string; Value int } diff --git a/server_utils/origin.go b/server_utils/origin.go index a8b508f0d..5c90634e1 100644 --- a/server_utils/origin.go +++ b/server_utils/origin.go @@ -32,7 +32,6 @@ import ( "github.com/mitchellh/mapstructure" "github.com/pkg/errors" log "github.com/sirupsen/logrus" - "github.com/spf13/viper" "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/param" @@ -401,7 +400,7 @@ func (b *BaseOrigin) handleExportsCfg(o Origin) error { log.Infoln("Configuring multi-exports from Origin.Exports block in config file") var tmpExports []OriginExport - if err := viper.UnmarshalKey(param.Origin_Exports.GetName(), &tmpExports, viper.DecodeHook(OriginExportsDecoderHook())); err != nil { + if err := param.Origin_Exports.UnmarshalWithHook(&tmpExports, OriginExportsDecoderHook()); err != nil { return errors.Wrap(err, "unable to parse the Origin.Exports configuration") } if len(tmpExports) == 0 { @@ -564,8 +563,6 @@ func GetOriginExports() ([]OriginExport, error) { return originExports, nil } - // This default also set in config.go, but duplicating it here makes testing a bit easier. - viper.SetDefault("Origin.StorageType", "posix") storageType, err := server_structs.ParseOriginStorageType(param.Origin_StorageType.GetString()) if err != nil { return originExports, err diff --git a/server_utils/origin_globus.go b/server_utils/origin_globus.go index 07e40d31d..7065177c4 100644 --- a/server_utils/origin_globus.go +++ b/server_utils/origin_globus.go @@ -23,7 +23,6 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" - "github.com/spf13/viper" "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" @@ -61,7 +60,7 @@ func (o *GlobusOrigin) validateExtra(e *OriginExport, numExports int) (err error return errors.Errorf("export %s sets the 'Listings' capability; listings are not yet supported for origins with %s of 'globus'", e.FederationPrefix, param.Origin_StorageType.GetName()) } - if viper.GetString(param.OIDC_Issuer.GetName()) != "globus" { + if param.OIDC_Issuer.GetString() != "globus" { clientIDFile := param.Origin_GlobusClientIDFile.GetString() if clientIDFile == "" { return errors.Errorf("%s is a required parameter for Globus origins when 'OIDC.Issuer' is not Globus", param.Origin_GlobusClientIDFile.GetName()) diff --git a/utils/utils.go b/utils/utils.go index 765c74cce..8c1c769c3 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -29,9 +29,10 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" - "github.com/spf13/viper" "golang.org/x/text/cases" "golang.org/x/text/language" + + "github.com/pelicanplatform/pelican/param" ) var ( @@ -189,8 +190,10 @@ func MapToSlice[K comparable, V any](m map[K]V) []V { // can validate relative values of different watermarks (e.g. !(low > high)). The returned float64 is only meant // to be used for comparing two watermark values, but only if both are either percentages or byte values, as // indicated by the isAbsolute return value. -func ValidateWatermark(paramName string, requireSuffix bool) (wm float64, isAbsolute bool, err error) { - wmStr := viper.GetString(paramName) +func ValidateWatermark(wmParam param.StringParam, requireSuffix bool) (wm float64, isAbsolute bool, err error) { + wmStr := wmParam.GetString() + paramName := wmParam.GetName() + if wmStr == "" { return 0, false, errors.Errorf("watermark value for config param '%s' is empty.", paramName) } diff --git a/utils/utils_test.go b/utils/utils_test.go index d88f8b038..df0371995 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -57,6 +57,10 @@ func TestValidateWatermark(t *testing.T) { defer func() { require.NoError(t, param.Reset()) }() t.Parallel() + + // Use a real parameter for testing + testParam := param.Cache_LowWatermark + testCases := []struct { name string wm string @@ -173,8 +177,8 @@ func TestValidateWatermark(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - require.NoError(t, param.Set(tc.name, tc.wm)) - val, isAbs, err := ValidateWatermark(tc.name, tc.requireSuffix) + require.NoError(t, param.Set(testParam.GetName(), tc.wm)) + val, isAbs, err := ValidateWatermark(testParam, tc.requireSuffix) if tc.expectErr { assert.Equal(t, 0.0, val) assert.False(t, isAbs)