Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -1711,11 +1711,11 @@ func InitServer(ctx context.Context, currentServers server_structs.ServerType) e
}

if param.Cache_LowWatermark.IsSet() || param.Cache_HighWaterMark.IsSet() {
lowWm, lwmIsAbs, err := utils.ValidateWatermark(param.Cache_LowWatermark.GetName(), false)
lowWm, lwmIsAbs, err := utils.ValidateWatermark(param.Cache_LowWatermark, false)
if err != nil {
return err
}
highWm, hwmIsAbs, err := utils.ValidateWatermark(param.Cache_HighWaterMark.GetName(), false)
highWm, hwmIsAbs, err := utils.ValidateWatermark(param.Cache_HighWaterMark, false)
if err != nil {
return err
}
Expand All @@ -1741,13 +1741,13 @@ func InitServer(ctx context.Context, currentServers server_structs.ServerType) e
var base, nominal, max float64
var err error
// Watermark validation will error if these parameters are not absolute, so we can ignore that output
if base, _, err = utils.ValidateWatermark(param.Cache_FilesBaseSize.GetName(), true); err != nil {
if base, _, err = utils.ValidateWatermark(param.Cache_FilesBaseSize, true); err != nil {
return err
}
if nominal, _, err = utils.ValidateWatermark(param.Cache_FilesNominalSize.GetName(), true); err != nil {
if nominal, _, err = utils.ValidateWatermark(param.Cache_FilesNominalSize, true); err != nil {
return err
}
if max, _, err = utils.ValidateWatermark(param.Cache_FilesMaxSize.GetName(), true); err != nil {
if max, _, err = utils.ValidateWatermark(param.Cache_FilesMaxSize, true); err != nil {
return err
}
if base >= nominal || nominal >= max {
Expand Down
2 changes: 2 additions & 0 deletions config/resources/defaults.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ Origin:
EnableMacaroons: false
EnableVoms: true
SelfTestInterval: 15s
StorageType: "posix"
Registry:
InstitutionsUrlReloadMinutes: 15m
RequireCacheApproval: false
Expand All @@ -109,6 +110,7 @@ Shoveler:
PortLower: 9930
PortHigher: 9999
AMQPExchange: shoveled-xrd
Topic: "xrootd.shoveler"
Xrootd:
MaxStartupWait: "10s"
Mount: ""
Expand Down
9 changes: 4 additions & 5 deletions director/maxmind.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ import (
"github.com/oschwald/geoip2-golang/v2"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"

"github.com/pelicanplatform/pelican/param"
"github.com/pelicanplatform/pelican/server_structs"
Expand Down Expand Up @@ -76,17 +75,17 @@ func downloadDB(localFile string) error {

var licenseKey string
keyFile := param.Director_MaxMindKeyFile.GetString()
keyFromEnv := viper.GetString("MAXMINDKEY")
keyFromParam := param.Director_MaxMindKey.GetString()
if keyFile != "" {
contents, err := os.ReadFile(keyFile)
if err != nil {
return err
}
licenseKey = strings.TrimSpace(string(contents))
} else if keyFromEnv != "" {
licenseKey = keyFromEnv
} else if keyFromParam != "" {
licenseKey = keyFromParam
} else {
return errors.New("A MaxMind key file must be specified in the config (Director.MaxMindKeyFile), in the environment (PELICAN_DIRECTOR_MAXMINDKEYFILE), or the key must be provided via the environment variable PELICAN_MAXMINDKEY)")
return errors.New("A MaxMind key file must be specified in the config (Director.MaxMindKeyFile), in the environment (PELICAN_DIRECTOR_MAXMINDKEYFILE), or the key must be provided via the environment variable PELICAN_DIRECTOR_MAXMINDKEY)")
}

url := fmt.Sprintf(maxMindURL, licenseKey)
Expand Down
9 changes: 9 additions & 0 deletions docs/parameters.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1981,6 +1981,15 @@ type: filename
default: none
components: ["director"]
---
name: Director.MaxMindKey
description: |+
A MaxMind API key provided directly as a string. This is an alternative to providing the key via a file
(Director.MaxMindKeyFile). If both are set, the file takes precedence. This parameter can be set via the
environment variable PELICAN_DIRECTOR_MAXMINDKEY.
type: string
default: none
components: ["director"]
---
name: Director.GeoIPLocation
description: |+
A filepath to the intended location of the MaxMind GeoLite City database. This option can be used either to load
Expand Down
3 changes: 1 addition & 2 deletions lotman/lotman_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ import (
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"

"github.com/pelicanplatform/pelican/config"
"github.com/pelicanplatform/pelican/param"
Expand Down Expand Up @@ -483,7 +482,7 @@ func GetPolicyMap() (map[string]PurgePolicy, error) {
policyMap := make(map[string]PurgePolicy)
var policies []PurgePolicy
// Use custom decoder hook to validate fields. This validates all the way down to the bottom of the lot object.
if err := viper.UnmarshalKey(param.Lotman_PolicyDefinitions.GetName(), &policies, viper.DecodeHook(validateFieldsHook())); err != nil {
if err := param.Lotman_PolicyDefinitions.UnmarshalWithHook(&policies, validateFieldsHook()); err != nil {
return policyMap, errors.Wrap(err, "error unmarshalling Lotman policy definitions")
}

Expand Down
3 changes: 0 additions & 3 deletions metrics/shoveler.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ import (
shoveler "github.com/opensciencegrid/xrootd-monitoring-shoveler"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
"golang.org/x/sync/errgroup"

"github.com/pelicanplatform/pelican/config"
Expand Down Expand Up @@ -88,8 +87,6 @@ func configShoveler(c *shoveler.Config) error {
return fmt.Errorf("Token content is empty. Reading from Shoveler.AMQPTokenLocation at %s", c.AmqpToken)
}
} else { // Stomp
viper.SetDefault("Shoveler.Topic", "xrootd.shoveler")

c.StompUser = param.Shoveler_StompUsername.GetString()
log.Debugln("STOMP User:", c.StompUser)
c.StompPassword = param.Shoveler_StompPassword.GetString()
Expand Down
9 changes: 9 additions & 0 deletions param/parameters.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions param/parameters_struct.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 1 addition & 4 deletions server_utils/origin.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ import (
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"

"github.com/pelicanplatform/pelican/config"
"github.com/pelicanplatform/pelican/param"
Expand Down Expand Up @@ -401,7 +400,7 @@ func (b *BaseOrigin) handleExportsCfg(o Origin) error {

log.Infoln("Configuring multi-exports from Origin.Exports block in config file")
var tmpExports []OriginExport
if err := viper.UnmarshalKey(param.Origin_Exports.GetName(), &tmpExports, viper.DecodeHook(OriginExportsDecoderHook())); err != nil {
if err := param.Origin_Exports.UnmarshalWithHook(&tmpExports, OriginExportsDecoderHook()); err != nil {
return errors.Wrap(err, "unable to parse the Origin.Exports configuration")
}
if len(tmpExports) == 0 {
Expand Down Expand Up @@ -564,8 +563,6 @@ func GetOriginExports() ([]OriginExport, error) {
return originExports, nil
}

// This default also set in config.go, but duplicating it here makes testing a bit easier.
viper.SetDefault("Origin.StorageType", "posix")
storageType, err := server_structs.ParseOriginStorageType(param.Origin_StorageType.GetString())
if err != nil {
return originExports, err
Expand Down
3 changes: 1 addition & 2 deletions server_utils/origin_globus.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (

"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"

"github.com/pelicanplatform/pelican/param"
"github.com/pelicanplatform/pelican/server_structs"
Expand Down Expand Up @@ -61,7 +60,7 @@ func (o *GlobusOrigin) validateExtra(e *OriginExport, numExports int) (err error
return errors.Errorf("export %s sets the 'Listings' capability; listings are not yet supported for origins with %s of 'globus'", e.FederationPrefix, param.Origin_StorageType.GetName())
}

if viper.GetString(param.OIDC_Issuer.GetName()) != "globus" {
if param.OIDC_Issuer.GetString() != "globus" {
clientIDFile := param.Origin_GlobusClientIDFile.GetString()
if clientIDFile == "" {
return errors.Errorf("%s is a required parameter for Globus origins when 'OIDC.Issuer' is not Globus", param.Origin_GlobusClientIDFile.GetName())
Expand Down
9 changes: 6 additions & 3 deletions utils/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,10 @@ import (

"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
"golang.org/x/text/cases"
"golang.org/x/text/language"

"github.com/pelicanplatform/pelican/param"
)

var (
Expand Down Expand Up @@ -189,8 +190,10 @@ func MapToSlice[K comparable, V any](m map[K]V) []V {
// can validate relative values of different watermarks (e.g. !(low > high)). The returned float64 is only meant
// to be used for comparing two watermark values, but only if both are either percentages or byte values, as
// indicated by the isAbsolute return value.
func ValidateWatermark(paramName string, requireSuffix bool) (wm float64, isAbsolute bool, err error) {
wmStr := viper.GetString(paramName)
func ValidateWatermark(wmParam param.StringParam, requireSuffix bool) (wm float64, isAbsolute bool, err error) {
wmStr := wmParam.GetString()
paramName := wmParam.GetName()

if wmStr == "" {
return 0, false, errors.Errorf("watermark value for config param '%s' is empty.", paramName)
}
Expand Down
8 changes: 6 additions & 2 deletions utils/utils_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ func TestValidateWatermark(t *testing.T) {
defer func() { require.NoError(t, param.Reset()) }()

t.Parallel()

// Use a real parameter for testing
testParam := param.Cache_LowWatermark

testCases := []struct {
name string
wm string
Expand Down Expand Up @@ -173,8 +177,8 @@ func TestValidateWatermark(t *testing.T) {

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
require.NoError(t, param.Set(tc.name, tc.wm))
val, isAbs, err := ValidateWatermark(tc.name, tc.requireSuffix)
require.NoError(t, param.Set(testParam.GetName(), tc.wm))
val, isAbs, err := ValidateWatermark(testParam, tc.requireSuffix)
if tc.expectErr {
assert.Equal(t, 0.0, val)
assert.False(t, isAbs)
Expand Down