diff --git a/config/resources/defaults.yaml b/config/resources/defaults.yaml index 8f4d49653..1f3190fcf 100644 --- a/config/resources/defaults.yaml +++ b/config/resources/defaults.yaml @@ -59,6 +59,7 @@ Director: AssumePresenceAtSingleOrigin: true CachePresenceTTL: 1m CachePresenceCapacity: 10000 + ProhibitedCachesRefreshInterval: 1m Cache: Port: 8442 SelfTest: true diff --git a/director/advertise_test.go b/director/advertise_test.go index 3cd3ddb5f..72be9d0b9 100644 --- a/director/advertise_test.go +++ b/director/advertise_test.go @@ -223,6 +223,9 @@ func TestAdvertiseOSDF(t *testing.T) { require.NotNil(t, foundServer.NamespaceAds) assert.True(t, foundServer.NamespaceAds[0].FromTopology) + // Override the check for prohibited caches + prohibitedCachesLastSetTimestamp.Store(time.Now().Unix()) + // Test a few values. If they're correct, it indicates the whole process likely succeeded nsAd, oAds, cAds := getAdsForPath("/my/server/path/to/file") assert.Equal(t, "/my/server", nsAd.Path) diff --git a/director/cache_ads.go b/director/cache_ads.go index 9b19cc686..6dc667963 100644 --- a/director/cache_ads.go +++ b/director/cache_ads.go @@ -304,6 +304,51 @@ func matchesPrefix(reqPath string, namespaceAds []server_structs.NamespaceAdV2) return best } +// isProhibited checks if the given server is prohibited from serving +// the specified namespace. It validates this by referencing the +// prohibitedCaches map that the director maintains in memory. +func isProhibited(nsAd *server_structs.NamespaceAdV2, ad *server_structs.Advertisement) bool { + if ad.Type == server_structs.OriginType.String() { + return false + } + if prohibitedCachesLastSetTimestamp.Load() == 0 { + log.Warning("Prohibited caches data is not set, waiting for it to be set before continuing with cache matchmaking") + start := time.Now() + // Wait until last set timestamp is updated + for prohibitedCachesLastSetTimestamp.Load() == 0 { + if time.Since(start) >= 3*time.Second { + log.Error("Prohibited caches data was not set within the 3-second timeout") + break + } + time.Sleep(100 * time.Millisecond) + } + } + if time.Since(time.Unix(prohibitedCachesLastSetTimestamp.Load(), 0)) >= 15*time.Minute { + log.Error("Prohibited caches data is outdated, caches will not be used.") + return true + } + + serverHost := ad.ServerAd.URL.Host + serverHostname := strings.Split(serverHost, ":")[0] + + prohibitedCachesData := prohibitedCaches.Load() + if prohibitedCachesData == nil { + return false + } + + for prefix, caches := range *prohibitedCachesData { + if strings.HasPrefix(nsAd.Path, prefix) { + for _, cacheHostname := range caches { + if strings.EqualFold(serverHostname, cacheHostname) { + return true + } + } + } + } + + return false +} + func getAdsForPath(reqPath string) (originNamespace server_structs.NamespaceAdV2, originAds []server_structs.ServerAd, cacheAds []server_structs.ServerAd) { skippedServers := []server_structs.ServerAd{} @@ -326,7 +371,7 @@ func getAdsForPath(reqPath string) (originNamespace server_structs.NamespaceAdV2 log.Debugf("Skipping %s server %s as it's in the filtered server list with type %s", ad.Type, ad.Name, ft) continue } - if ns := matchesPrefix(reqPath, ad.NamespaceAds); ns != nil { + if ns := matchesPrefix(reqPath, ad.NamespaceAds); ns != nil && !isProhibited(ns, ad) { if best == nil || len(ns.Path) > len(best.Path) { best = ns // If anything was previously set by a namespace that constituted a shorter diff --git a/director/cache_ads_test.go b/director/cache_ads_test.go index 15699834c..882272e65 100644 --- a/director/cache_ads_test.go +++ b/director/cache_ads_test.go @@ -159,6 +159,9 @@ func TestGetAdsForPath(t *testing.T) { recordAd(context.Background(), cacheAd1, &c1Slice) recordAd(context.Background(), cacheAd2, &o1Slice) + // Override the check for prohibited caches + prohibitedCachesLastSetTimestamp.Store(time.Now().Unix()) + // If /chtc is served both from topology and Pelican, the Topology server/namespace should be ignored nsAd, oAds, cAds := getAdsForPath("/chtc") assert.Equal(t, "/chtc", nsAd.Path) @@ -210,6 +213,19 @@ func TestGetAdsForPath(t *testing.T) { assert.Equal(t, 0, len(oAds)) assert.Equal(t, 0, len(cAds)) + // Prohibited caches should not be matched + prohibitedCaches.Store(&map[string][]string{ + "/chtc": {"cache1.wisc.edu"}, + }) + + nsAd, oAds, cAds = getAdsForPath("/chtc/PUBLIC") + assert.Equal(t, 1, len(oAds)) + assert.Equal(t, 0, len(cAds)) + assert.True(t, hasServerAdWithName(oAds, "origin2")) + assert.False(t, hasServerAdWithName(cAds, "cache1")) + + prohibitedCaches.Store(&map[string][]string{}) + // Filtered server should not be included filteredServersMutex.Lock() tmp := filteredServers diff --git a/director/prohibited_caches.go b/director/prohibited_caches.go new file mode 100644 index 000000000..edf10f3ed --- /dev/null +++ b/director/prohibited_caches.go @@ -0,0 +1,147 @@ +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + +package director + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "sync/atomic" + "time" + + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "golang.org/x/sync/errgroup" + + "github.com/pelicanplatform/pelican/config" + "github.com/pelicanplatform/pelican/param" +) + +var ( + // prohibitedCaches maps federation prefixes to a list of cache hostnames where data should not propagate. + prohibitedCaches atomic.Pointer[map[string][]string] + // prohibitedCachesLastSetTimestamp tracks when prohibitedCaches was last explicitly set. + prohibitedCachesLastSetTimestamp atomic.Int64 +) + +func init() { + emptyMap := make(map[string][]string) + prohibitedCaches.Store(&emptyMap) + + // Initialize prohibitedCachesLastSetTimestamp to 0 (indicating never set) + prohibitedCachesLastSetTimestamp.Store(0) +} + +// fetchProhibitedCaches makes a request to the registry endpoint to retrieve +// information about prohibited caches and returns the result. +func fetchProhibitedCaches(ctx context.Context) (map[string][]string, error) { + fedInfo, err := config.GetFederation(ctx) + if err != nil { + return nil, err + } + registryUrlStr := fedInfo.RegistryEndpoint + registryUrl, err := url.Parse(registryUrlStr) + if err != nil { + return nil, err + } + reqUrl := registryUrl.JoinPath("/api/v1.0/registry/namespaces/prohibitedCaches") + + client := http.Client{Transport: config.GetTransport()} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqUrl.String(), nil) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", "application/json") + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to fetch prohibited caches from the registry: unexpected status code %d", resp.StatusCode) + } + + var result map[string][]string + err = json.NewDecoder(resp.Body).Decode(&result) + if err != nil { + return nil, err + } + + return result, nil +} + +// LaunchPeriodicProhibitedCachesFetch starts a new goroutine that periodically +// refreshes the prohibited cache data maintained by the director in memory. +func LaunchPeriodicProhibitedCachesFetch(ctx context.Context, egrp *errgroup.Group) { + refreshInterval := param.Director_ProhibitedCachesRefreshInterval.GetDuration() + + if refreshInterval < 1*time.Millisecond { + log.Warnf("Director.ProhibitedCachesRefreshInterval is set to: %v, which is too low. Falling back to default: 1m", refreshInterval) + + viper.Set("Director.ProhibitedCachesRefreshInterval", "1m") + refreshInterval = 1 * time.Minute + } + + ticker := time.NewTicker(refreshInterval) + + // Initial fetch of prohibited caches + data, err := fetchProhibitedCaches(ctx) + if err != nil { + ticker.Reset(1 * time.Second) // Higher frequency (10s) + log.Warningf("Error fetching prohibited caches on first attempt: %v", err) + log.Debug("Switching to higher frequency (1s) for prohibited caches fetch") + } else { + prohibitedCaches.Store(&data) + prohibitedCachesLastSetTimestamp.Store(time.Now().Unix()) + log.Debug("Prohibited caches updated successfully on first attempt") + } + + egrp.Go(func() error { + defer ticker.Stop() + + for { + select { + case <-ticker.C: + // Fetch the prohibited caches + data, err := fetchProhibitedCaches(ctx) + if err != nil { + log.Warningf("Error fetching prohibited caches: %v", err) + lastSet := prohibitedCachesLastSetTimestamp.Load() + if time.Since(time.Unix(lastSet, 0)) >= 15*time.Minute { + log.Debug("Prohibited caches last updated over 15 minutes ago, switching to higher frequency") + ticker.Reset(1 * time.Second) // Higher frequency (1s) + } + continue + } + ticker.Reset(refreshInterval) // Normal frequency + prohibitedCaches.Store(&data) + prohibitedCachesLastSetTimestamp.Store(time.Now().Unix()) + log.Debug("Prohibited caches updated successfully") + case <-ctx.Done(): + log.Debug("Periodic fetch terminated") + return nil + } + } + }) +} diff --git a/director/prohibited_caches_test.go b/director/prohibited_caches_test.go new file mode 100644 index 000000000..0cbdb4440 --- /dev/null +++ b/director/prohibited_caches_test.go @@ -0,0 +1,98 @@ +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + +package director + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + + "github.com/pelicanplatform/pelican/config" +) + +// TestLaunchPeriodicProhibitedCachesFetch tests whether the prohibited caches data is periodically updated from the registry. +func TestLaunchPeriodicProhibitedCachesFetch(t *testing.T) { + config.ResetConfig() + defer config.ResetConfig() + + mockDataChan := make(chan map[string][]string, 2) + mockData := map[string][]string{ + "/foo/bar": {"hostname5", "hostname6"}, + } + mockDataChan <- mockData + + var lastData map[string][]string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + select { + case currentData := <-mockDataChan: + lastData = currentData + default: + } + if lastData == nil { + lastData = make(map[string][]string) + } + if err := json.NewEncoder(w).Encode(lastData); err != nil { + log.Errorf("Failed to encode response: %v", err) + } + })) + defer server.Close() + + // Set the registry URL to the mock server. + viper.Set("Federation.Registryurl", server.URL) + viper.Set("Director.ProhibitedCachesRefreshInterval", "200ms") + + ctx, cancel := context.WithCancel(context.Background()) + egrp := &errgroup.Group{} + + LaunchPeriodicProhibitedCachesFetch(ctx, egrp) + + time.Sleep(500 * time.Millisecond) + + currentMapPtr := prohibitedCaches.Load() + assert.NotNil(t, currentMapPtr, "prohibitedCaches should not be nil") + + assert.Equal(t, mockData, *currentMapPtr, "prohibitedCaches does not match the expected value") + + mockData = map[string][]string{ + "/foo/bar": {"hostname3", "hostname4"}, + } + mockDataChan <- mockData + + time.Sleep(500 * time.Millisecond) + + currentMapPtr = prohibitedCaches.Load() + assert.NotNil(t, currentMapPtr, "prohibitedCaches should not be nil") + + assert.Equal(t, mockData, *currentMapPtr, "prohibitedCaches does not match the expected value") + + cancel() + + require.NoError(t, egrp.Wait(), "Periodic fetch goroutine did not terminate properly") +} diff --git a/docs/parameters.yaml b/docs/parameters.yaml index 028a661d2..70292a342 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -1516,6 +1516,14 @@ default: 10000 hidden: true components: ["director"] --- +name: Director.ProhibitedCachesRefreshInterval +description: |+ + Specifies the time interval after which the director queries the registry to refresh its in-memory data about prohibited caches. + Prohibited caches data is essentially a mapping of federation prefixes to a list of caches to which the data should not be propagated for the given prefix. +type: duration +default: 1min +components: ["director"] +--- ############################ # Registry-level configs # ############################ diff --git a/launchers/director_serve.go b/launchers/director_serve.go index 4fde00fc8..a9dc0a5c8 100644 --- a/launchers/director_serve.go +++ b/launchers/director_serve.go @@ -52,6 +52,8 @@ func DirectorServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group director.LaunchServerIOQuery(ctx, egrp) + director.LaunchPeriodicProhibitedCachesFetch(ctx, egrp) + if config.GetPreferredPrefix() == config.OsdfPrefix { metrics.SetComponentHealthStatus(metrics.DirectorRegistry_Topology, metrics.StatusWarning, "Start requesting from topology, status unknown") log.Info("Generating/advertising server ads from OSG topology service...") diff --git a/param/parameters.go b/param/parameters.go index f915935ef..6a61322ef 100644 --- a/param/parameters.go +++ b/param/parameters.go @@ -385,6 +385,7 @@ var ( Director_AdvertisementTTL = DurationParam{"Director.AdvertisementTTL"} Director_CachePresenceTTL = DurationParam{"Director.CachePresenceTTL"} Director_OriginCacheHealthTestInterval = DurationParam{"Director.OriginCacheHealthTestInterval"} + Director_ProhibitedCachesRefreshInterval = DurationParam{"Director.ProhibitedCachesRefreshInterval"} Director_StatTimeout = DurationParam{"Director.StatTimeout"} Federation_TopologyReloadInterval = DurationParam{"Federation.TopologyReloadInterval"} Monitoring_DataRetention = DurationParam{"Monitoring.DataRetention"} diff --git a/param/parameters_struct.go b/param/parameters_struct.go index 8f9165ca1..0f304765c 100644 --- a/param/parameters_struct.go +++ b/param/parameters_struct.go @@ -81,6 +81,7 @@ type Config struct { MinStatResponse int `mapstructure:"minstatresponse" yaml:"MinStatResponse"` OriginCacheHealthTestInterval time.Duration `mapstructure:"origincachehealthtestinterval" yaml:"OriginCacheHealthTestInterval"` OriginResponseHostnames []string `mapstructure:"originresponsehostnames" yaml:"OriginResponseHostnames"` + ProhibitedCachesRefreshInterval time.Duration `mapstructure:"prohibitedcachesrefreshinterval" yaml:"ProhibitedCachesRefreshInterval"` StatConcurrencyLimit int `mapstructure:"statconcurrencylimit" yaml:"StatConcurrencyLimit"` StatTimeout time.Duration `mapstructure:"stattimeout" yaml:"StatTimeout"` SupportContactEmail string `mapstructure:"supportcontactemail" yaml:"SupportContactEmail"` @@ -385,6 +386,7 @@ type configWithType struct { MinStatResponse struct { Type string; Value int } OriginCacheHealthTestInterval struct { Type string; Value time.Duration } OriginResponseHostnames struct { Type string; Value []string } + ProhibitedCachesRefreshInterval struct { Type string; Value time.Duration } StatConcurrencyLimit struct { Type string; Value int } StatTimeout struct { Type string; Value time.Duration } SupportContactEmail struct { Type string; Value string } diff --git a/registry/custom_reg_fields.go b/registry/custom_reg_fields.go index 58c7d3559..5bd49d18e 100644 --- a/registry/custom_reg_fields.go +++ b/registry/custom_reg_fields.go @@ -167,11 +167,12 @@ func excludePubKey(nss []server_structs.Namespace) (nssNew []NamespaceWOPubkey) nssNew = make([]NamespaceWOPubkey, 0) for _, ns := range nss { nsNew := NamespaceWOPubkey{ - ID: ns.ID, - Prefix: ns.Prefix, - Pubkey: ns.Pubkey, - AdminMetadata: ns.AdminMetadata, - Identity: ns.Identity, + ID: ns.ID, + Prefix: ns.Prefix, + Pubkey: ns.Pubkey, + AdminMetadata: ns.AdminMetadata, + Identity: ns.Identity, + ProhibitedCaches: ns.ProhibitedCaches, } nssNew = append(nssNew, nsNew) } diff --git a/registry/migrations/20241122165844_add_prohibited_caches_to_namespace.sql b/registry/migrations/20241122165844_add_prohibited_caches_to_namespace.sql new file mode 100644 index 000000000..51461ee3c --- /dev/null +++ b/registry/migrations/20241122165844_add_prohibited_caches_to_namespace.sql @@ -0,0 +1,11 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE namespace +ADD COLUMN prohibited_caches TEXT DEFAULT '[]'; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE namespace +DROP COLUMN prohibited_caches; +-- +goose StatementEnd diff --git a/registry/registry.go b/registry/registry.go index 077ba917e..b33b03efc 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -895,6 +895,9 @@ func wildcardHandler(ctx *gin.Context) { ctx.JSON(http.StatusOK, nsCfg) return + } else if strings.HasSuffix(path, "/namespaces/prohibitedCaches") { + getProhibitedCachesHandler(ctx) + return } else { // Default to get the namespace by its prefix getNamespaceHandler(ctx) @@ -1154,6 +1157,21 @@ func checkStatusHandler(ctx *gin.Context) { ctx.JSON(http.StatusOK, server_structs.CheckNamespaceCompleteRes{Results: results}) } +// getProhibitedCachesHandler is a wrapper around the getProhibitedCaches function. +func getProhibitedCachesHandler(ctx *gin.Context) { + caches, err := getProhibitedCaches() + + if err != nil { + ctx.JSON(http.StatusInternalServerError, server_structs.SimpleApiResp{ + Status: server_structs.RespFailed, + Msg: fmt.Sprintf("Error fetching prohibited caches: %s", err.Error()), + }) + return + } + + ctx.JSON(http.StatusOK, caches) +} + func RegisterRegistryAPI(router *gin.RouterGroup) { registryAPI := router.Group("/api/v1.0/registry") diff --git a/registry/registry_db.go b/registry/registry_db.go index 3d8afb9bc..7aa2d2c60 100644 --- a/registry/registry_db.go +++ b/registry/registry_db.go @@ -38,11 +38,12 @@ import ( ) type NamespaceWOPubkey struct { - ID int `json:"id"` - Prefix string `json:"prefix"` - Pubkey string `json:"-"` // Don't include pubkey in this case - Identity string `json:"identity"` - AdminMetadata server_structs.AdminMetadata `json:"admin_metadata"` + ID int `json:"id"` + Prefix string `json:"prefix"` + Pubkey string `json:"-"` // Don't include pubkey in this case + Identity string `json:"identity"` + AdminMetadata server_structs.AdminMetadata `json:"admin_metadata"` + ProhibitedCaches []string `json:"prohibited_caches"` } type Topology struct { @@ -165,7 +166,7 @@ func namespaceBelongsToUserId(id int, userId string) (bool, error) { var result server_structs.Namespace err := db.First(&result, "id = ?", id).Error if errors.Is(err, gorm.ErrRecordNotFound) { - return false, fmt.Errorf("Namespace with id = %d does not exists", id) + return false, fmt.Errorf("namespace with id = %d does not exists", id) } else if err != nil { return false, errors.Wrap(err, "error retrieving namespace") } @@ -280,6 +281,28 @@ func getNamespaceByPrefix(prefix string) (*server_structs.Namespace, error) { return &ns, nil } +// getProhibitedCaches queries the database and returns a map where the keys +// are federation prefixes and the values are lists of prohibited cache hostnames. +// Only prefixes with at least one prohibited cache are included in the resulting map. +func getProhibitedCaches() (map[string][]string, error) { + var namespaces []server_structs.Namespace + prohibitedCacheMap := make(map[string][]string) + + err := db.Select("prefix, prohibited_caches"). + Find(&namespaces).Error + if err != nil { + return nil, errors.Wrap(err, "error retrieving prohibited caches") + } + + for _, ns := range namespaces { + if len(ns.ProhibitedCaches) > 0 { + prohibitedCacheMap[ns.Prefix] = ns.ProhibitedCaches + } + } + + return prohibitedCacheMap, nil +} + // Get a collection of namespaces by filtering against various non-default namespace fields // excluding Namespace.ID, Namespace.Identity, Namespace.Pubkey, and various dates // diff --git a/registry/registry_db_test.go b/registry/registry_db_test.go index 402887f0e..099d18f6b 100644 --- a/registry/registry_db_test.go +++ b/registry/registry_db_test.go @@ -180,6 +180,8 @@ var ( "key2": 2, "key3": true, } + + mockProhibitedCaches = []string{"hostname1", "hostname2"} ) func TestGetNamespaceById(t *testing.T) { @@ -203,6 +205,7 @@ func TestGetNamespaceById(t *testing.T) { defer resetNamespaceDB(t) mockNs := mockNamespace("/test", "", "", server_structs.AdminMetadata{UserID: "foo"}) mockNs.CustomFields = mockCustomFields + mockNs.ProhibitedCaches = mockProhibitedCaches err := insertMockDBData([]server_structs.Namespace{mockNs}) require.NoError(t, err) nss, err := getAllNamespaces() @@ -317,6 +320,7 @@ func TestAddNamespace(t *testing.T) { SecurityContactUserID: "security-001", }) mockNs.CustomFields = mockCustomFields + mockNs.ProhibitedCaches = mockProhibitedCaches err := AddNamespace(&mockNs) require.NoError(t, err) got, err := getAllNamespaces() @@ -329,6 +333,8 @@ func TestAddNamespace(t *testing.T) { assert.Equal(t, mockNs.AdminMetadata.SiteName, got[0].AdminMetadata.SiteName) assert.Equal(t, mockNs.AdminMetadata.SecurityContactUserID, got[0].AdminMetadata.SecurityContactUserID) assert.Equal(t, mockCustomFields, got[0].CustomFields) + assert.Equal(t, mockProhibitedCaches, got[0].ProhibitedCaches) + }) } diff --git a/registry/registry_test.go b/registry/registry_test.go index 43ad3d7e0..b9efb3024 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -190,6 +190,26 @@ func TestHandleWildcard(t *testing.T) { } }) } + + t.Run("match-output-of-prohibited-caches-endpoint", func(t *testing.T) { + setupMockRegistryDB(t) + defer teardownMockNamespaceDB(t) + + err := insertMockDBData([]server_structs.Namespace{{Prefix: "/foo/bar", ProhibitedCaches: []string{"hostname1", "hostname2"}}}) + require.NoError(t, err) + + req, _ := http.NewRequest("GET", "/registry/namespaces/prohibitedCaches", nil) + w := httptest.NewRecorder() + + r.ServeHTTP(w, req) + assert.Equal(t, http.StatusOK, w.Code) + + expectedJSON := `{ + "/foo/bar": ["hostname1", "hostname2"] + }` + + assert.JSONEq(t, expectedJSON, w.Body.String(), "Response JSON does not match the expected output") + }) } func TestCheckNamespaceCompleteHandler(t *testing.T) { diff --git a/registry/registry_validation.go b/registry/registry_validation.go index 5b07b22dc..939d4d940 100644 --- a/registry/registry_validation.go +++ b/registry/registry_validation.go @@ -293,7 +293,7 @@ func validateCustomFields(customFields map[string]interface{}) (bool, error) { // Check the provided option is in the list of available options if len(options) == 0 { - return false, fmt.Errorf("Bad configuration, the custom field %q has empty options", regField.DisplayedName) + return false, fmt.Errorf("bad configuration, the custom field %q has empty options", regField.DisplayedName) } inOpt := false for _, item := range options { diff --git a/registry/registry_validation_test.go b/registry/registry_validation_test.go index 469844e99..61bbefc6b 100644 --- a/registry/registry_validation_test.go +++ b/registry/registry_validation_test.go @@ -114,7 +114,7 @@ func TestValidateCustomFields(t *testing.T) { valid, err := validateCustomFields(customFields) require.Error(t, err, "Expected an error due to invalid enum value") - assert.Contains(t, err.Error(), `Bad configuration, the custom field "Enum Field" has empty options`) + assert.Contains(t, err.Error(), `bad configuration, the custom field "Enum Field" has empty options`) assert.False(t, valid, "Validation should fail due to invalid enum value") }) t.Run("enum-field-with-optionsUrl", func(t *testing.T) { diff --git a/server_structs/registry.go b/server_structs/registry.go index f30415d93..87583d0a5 100644 --- a/server_structs/registry.go +++ b/server_structs/registry.go @@ -21,6 +21,8 @@ package server_structs import ( "strings" "time" + + "gorm.io/gorm" ) type RegistrationStatus string @@ -54,12 +56,13 @@ type AdminMetadata struct { } type Namespace struct { - ID int `json:"id" post:"exclude" gorm:"primaryKey"` - Prefix string `json:"prefix" validate:"required"` - Pubkey string `json:"pubkey" validate:"required" description:"Pubkey is your Pelican server public key in JWKS form"` - Identity string `json:"identity" post:"exclude"` - AdminMetadata AdminMetadata `json:"admin_metadata" gorm:"serializer:json"` - CustomFields map[string]interface{} `json:"custom_fields" gorm:"serializer:json"` + ID int `json:"id" post:"exclude" gorm:"primaryKey"` + Prefix string `json:"prefix" validate:"required"` + Pubkey string `json:"pubkey" validate:"required" description:"Pubkey is your Pelican server public key in JWKS form"` + Identity string `json:"identity" post:"exclude"` + AdminMetadata AdminMetadata `json:"admin_metadata" gorm:"serializer:json"` + ProhibitedCaches []string `json:"prohibited_caches" gorm:"serializer:json"` + CustomFields map[string]interface{} `json:"custom_fields" gorm:"serializer:json"` } type ( @@ -133,3 +136,12 @@ func (Namespace) TableName() string { func IsValidRegStatus(s string) bool { return s == "Pending" || s == "Approved" || s == "Denied" || s == "Unknown" } + +// GORM hook to set ProhibitedCaches to an empty slice instead of nil before saving (create or update). +// This ensures that when converted to JSON, it appears as [] instead of null. +func (n *Namespace) BeforeSave(tx *gorm.DB) (err error) { + if n.ProhibitedCaches == nil { + n.ProhibitedCaches = []string{} + } + return nil +} diff --git a/swagger/pelican-swagger.yaml b/swagger/pelican-swagger.yaml index 55dc1a258..31247c3ff 100644 --- a/swagger/pelican-swagger.yaml +++ b/swagger/pelican-swagger.yaml @@ -197,6 +197,11 @@ definitions: description: The user identity we get from CILogon if the namespace is registered via CLI with `--with-identity` flag admin_metadata: $ref: "#/definitions/AdminMetadata" + prohibited_caches: + type: array + items: + type: string + description: A list of cache hostnames where data for this namespace prefix should not propagate. custom_fields: type: object description: The custom fields user registered, configurable by setting Registry.CustomRegistrationFields. @@ -232,6 +237,11 @@ definitions: It should be a marshaled (stringfied) JSON that contains either one JWK or a JWKS admin_metadata: $ref: "#/definitions/AdminMetadata" + prohibited_caches: + type: array + items: + type: string + description: A list of cache hostnames where data for this namespace prefix should not propagate. custom_fields: type: object description: The custom fields user registered, configurable by setting Registry.CustomRegistrationFields.