diff --git a/.github/workflows/test-template.yml b/.github/workflows/test-template.yml new file mode 100644 index 000000000..0109c967e --- /dev/null +++ b/.github/workflows/test-template.yml @@ -0,0 +1,101 @@ +name: Test Template + +on: + workflow_call: + inputs: + tags: + required: true + type: string + coverprofile: + required: true + type: string + binary_name: + required: true + type: string + +jobs: + test: + runs-on: ubuntu-latest + container: + image: hub.opensciencegrid.org/pelican_platform/pelican-dev:latest-itb + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-node@v4 + with: + node-version: 20 + - name: Fetch tags + run: | + git config --global --add safe.directory /__w/pelican/pelican + git fetch --force --tags + - name: Cache Next.js + uses: actions/cache@v4 + with: + path: | + ~/.npm + ${{ github.workspace }}/.next/cache + key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.js', '**/*.jsx', '**/*.ts', '**/*.tsx') }} + restore-keys: | + ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}- + - name: Test + run: | + make web-build + go test -tags=${{ inputs.tags }} -timeout 15m -coverpkg=./... -coverprofile=${{ inputs.coverprofile }} -covermode=count ./... + - name: Get total code coverage + if: github.event_name == 'pull_request' + id: cc + run: | + set -x + cc_total=`go tool cover -func=${{ inputs.coverprofile }} | grep total | grep -Eo '[0-9]+\.[0-9]+'` + echo "cc_total=$cc_total" >> $GITHUB_OUTPUT + - name: Restore base test coverage + id: base-coverage + if: github.event.pull_request.base.sha != '' + uses: actions/cache@v4 + with: + path: | + unit-base.txt + key: ${{ runner.os }}-unit-test-coverage-${{ (github.event.pull_request.base.sha != github.event.after) && github.event.pull_request.base.sha || github.event.after }} + - name: Run test for base code + if: steps.base-coverage.outputs.cache-hit != 'true' && github.event.pull_request.base.sha != '' + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch origin main ${{ github.event.pull_request.base.sha }} + HEAD=$(git rev-parse HEAD) + git reset --hard ${{ github.event.pull_request.base.sha }} + make web-build + go generate ./... + go test -tags=${{ inputs.tags }} -timeout 15m -coverpkg=./... -coverprofile=base_coverage.out -covermode=count ./... + go tool cover -func=base_coverage.out > unit-base.txt + git reset --hard $HEAD + - name: Get base code coverage value + if: github.event_name == 'pull_request' + id: cc_b + run: | + set -x + cc_base_total=`grep total ./unit-base.txt | grep -Eo '[0-9]+\.[0-9]+'` + echo "cc_base_total=$cc_base_total" >> $GITHUB_OUTPUT + - name: Add coverage information to action summary + if: github.event_name == 'pull_request' + run: echo 'Code coverage ' ${{steps.cc.outputs.cc_total}}'% Prev ' ${{steps.cc_b.outputs.cc_base_total}}'%' >> $GITHUB_STEP_SUMMARY + - name: Run GoReleaser for Ubuntu + uses: goreleaser/goreleaser-action@v5 + with: + distribution: goreleaser + version: latest + args: build --single-target --clean --snapshot + - name: Copy files (Ubuntu) + run: | + cp dist/${{ inputs.binary_name }}_linux_amd64_v1/${{ inputs.binary_name }} ./pelican + - name: Run Integration Tests + run: ./github_scripts/citests.sh + - name: Run End-to-End Test for Object get/put + run: ./github_scripts/get_put_test.sh + - name: Run End-to-End Test for Director stat + run: ./github_scripts/stat_test.sh + - name: Run End-to-End Test of x509 access + run: ./github_scripts/x509_test.sh + - name: Run End-to-End Test for --version flag + run: ./github_scripts/version_test.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5adb86ac2..0804df90f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,8 +13,8 @@ jobs: - name: Checkout code uses: actions/checkout@v4 with: - # Do fetch depth 0 here because otherwise goreleaser might not work properly: - # https://goreleaser.com/ci/actions/?h=tag#workflow + # Do fetch depth 0 here because otherwise goreleaser might not work properly: + # https://goreleaser.com/ci/actions/?h=tag#workflow fetch-depth: 0 - uses: actions/setup-node@v4 with: @@ -55,94 +55,14 @@ jobs: version: latest args: build --single-target --clean --snapshot test-ubuntu: - runs-on: ubuntu-latest - container: - image: hub.opensciencegrid.org/pelican_platform/pelican-dev:latest-itb - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - # See above for why fetch depth is 0 here - fetch-depth: 0 - - uses: actions/setup-node@v4 - with: - node-version: 20 - # Fetch the tags is essential so that goreleaser can build the correct version. Workaround found here: - # https://github.com/actions/checkout/issues/290 - - name: Fetch tags - run: | - git config --global --add safe.directory /__w/pelican/pelican - git fetch --force --tags - - name: Cache Next.js - uses: actions/cache@v4 - with: - path: | - ~/.npm - ${{ github.workspace }}/.next/cache - # Generate a new cache whenever packages or source files change. - key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.js', '**/*.jsx', '**/*.ts', '**/*.tsx') }} - # If source files changed but packages didn't, rebuild from a prior cache. - restore-keys: | - ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}- - - name: Test - run: | - make web-build - go test -timeout 15m -coverpkg=./... -coverprofile=coverage.out -covermode=count ./... - - name: Get total code coverage - if: github.event_name == 'pull_request' - id: cc - run: | - set -x - cc_total=`go tool cover -func=coverage.out | grep total | grep -Eo '[0-9]+\.[0-9]+'` - echo "cc_total=$cc_total" >> $GITHUB_OUTPUT - - name: Restore base test coverage - id: base-coverage - if: github.event.pull_request.base.sha != '' - uses: actions/cache@v4 - with: - path: | - unit-base.txt - # Use base sha for PR or new commit hash for master/main push in test result key. - key: ${{ runner.os }}-unit-test-coverage-${{ (github.event.pull_request.base.sha != github.event.after) && github.event.pull_request.base.sha || github.event.after }} - - name: Run test for base code - if: steps.base-coverage.outputs.cache-hit != 'true' && github.event.pull_request.base.sha != '' - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git fetch origin main ${{ github.event.pull_request.base.sha }} - HEAD=$(git rev-parse HEAD) - git reset --hard ${{ github.event.pull_request.base.sha }} - make web-build - go generate ./... - go test -timeout 15m -coverpkg=./... -coverprofile=base_coverage.out -covermode=count ./... - go tool cover -func=base_coverage.out > unit-base.txt - git reset --hard $HEAD - - name: Get base code coverage value - if: github.event_name == 'pull_request' - id: cc_b - run: | - set -x - cc_base_total=`grep total ./unit-base.txt | grep -Eo '[0-9]+\.[0-9]+'` - echo "cc_base_total=$cc_base_total" >> $GITHUB_OUTPUT - - name: Add coverage information to action summary - if: github.event_name == 'pull_request' - run: echo 'Code coverage ' ${{steps.cc.outputs.cc_total}}'% Prev ' ${{steps.cc_b.outputs.cc_base_total}}'%' >> $GITHUB_STEP_SUMMARY - - name: Run GoReleaser for Ubuntu - uses: goreleaser/goreleaser-action@v5 - with: - # either 'goreleaser' (default) or 'goreleaser-pro' - distribution: goreleaser - version: latest - args: build --single-target --clean --snapshot - - name: Copy files (Ubuntu) - run: | - cp dist/pelican_linux_amd64_v1/pelican ./ - - name: Run Integration Tests - run: ./github_scripts/citests.sh - - name: Run End-to-End Test for Object get/put - run: ./github_scripts/get_put_test.sh - - name: Run End-to-End Test for Director stat - run: ./github_scripts/stat_test.sh - - name: Run End-to-End Test of x509 access - run: ./github_scripts/x509_test.sh - - name: Run End-to-End Test for --version flag - run: ./github_scripts/version_test.sh + uses: ./.github/workflows/test-template.yml + with: + tags: "" + coverprofile: "coverage.out" + binary_name: "pelican" + test-ubuntu-server: + uses: ./.github/workflows/test-template.yml + with: + tags: "lotman" + coverprofile: "coverage-server.out" + binary_name: "pelican-server" diff --git a/.goreleaser.yml b/.goreleaser.yml index dc184d93a..5ae8b446f 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -50,9 +50,30 @@ builds: goarch: ppc64le - goos: darwin goarch: ppc64le - + # Set things up to build a second server binary that enables Lotman. Eventually + # we'll also use this to filter which moduels are built into the binary. + - env: + - CGO_ENABLED=0 + goos: + - linux + goarch: + - "amd64" + - "arm64" + id: "pelican-server" + dir: ./cmd + binary: pelican-server + tags: + - forceposix + - lotman + ldflags: + - -s -w -X github.com/pelicanplatform/pelican/config.commit={{.Commit}} -X github.com/pelicanplatform/pelican/config.date={{.Date}} -X github.com/pelicanplatform/pelican/config.builtBy=goreleaser -X github.com/pelicanplatform/pelican/config.version={{.Version}} +# Goreleaser complains if there's a different number of binaries built for different architectures +# in the same archive. Instead of plopping pelican-server in the same archive as pelican, split the +# builds into separate archives. archives: - id: pelican + builds: + - pelican name_template: >- {{ .ProjectName }}_ {{- title .Os }}_ @@ -62,6 +83,15 @@ archives: - goos: windows format: zip wrap_in_directory: '{{ .ProjectName }}-{{ trimsuffix .Version "-next" }}' + - id: pelican-server + builds: + - pelican-server + name_template: >- + {{ .ProjectName }}-server_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else }}{{ .Arch }}{{ end }} + wrap_in_directory: '{{ .ProjectName }}-server-{{ trimsuffix .Version "-next" }}' checksum: name_template: 'checksums.txt' snapshot: diff --git a/images/Dockerfile b/images/Dockerfile index 4689158fd..30836869f 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -220,8 +220,11 @@ RUN rm -rf /pelican/pelican #################### FROM pelican-base AS cache - -ENTRYPOINT [ "/entrypoint.sh", "pelican", "cache"] +RUN rm -rf /pelican/pelican +COPY --from=pelican-build /pelican/dist/pelican-server_linux_amd64_v1/pelican-server /pelican/pelican-server +RUN chmod +x /pelican/pelican-server +# For now, we're only using pelican-server in the cache, but eventually we'll use it in all servers +ENTRYPOINT [ "/entrypoint.sh", "pelican-server", "cache"] CMD [ "serve" ] #################### @@ -260,8 +263,10 @@ CMD [ "serve" ] #################### FROM osdf-base AS osdf-cache - -ENTRYPOINT [ "/entrypoint.sh" ,"osdf", "cache"] +RUN rm -rf /pelican/osdf +COPY --from=pelican-build /pelican/dist/pelican-server_linux_amd64_v1/pelican-server /pelican/osdf-server +RUN chmod +x /pelican/osdf-server +ENTRYPOINT [ "/entrypoint.sh" ,"osdf-server", "cache"] CMD [ "serve" ] #################### diff --git a/images/entrypoint.sh b/images/entrypoint.sh index d7f5943de..4d8b24037 100644 --- a/images/entrypoint.sh +++ b/images/entrypoint.sh @@ -119,6 +119,15 @@ if [ $# -ne 0 ]; then echo >&2 "Exec of tini failed!" exit 1 ;; + pelican-server) + # Our server-specific binary which may come with additional + # features/system requirements (like Lotman) + echo "Running pelican-server with arguments: $@" + exec tini -- /pelican/pelican-server "$@" + # we shouldn't get here + echo >&2 "Exec of tini failed!" + exit 1 + ;; osdf) # Run osdf with the rest of the arguments echo "Running osdf with arguments: $@" @@ -127,6 +136,13 @@ if [ $# -ne 0 ]; then echo >&2 "Exec of tini failed!" exit 1 ;; + osdf-server) + echo "Running osdf-server with arguments: $@" + exec tini -- /pelican/osdf-server "$@" + # we shouldn't get here + echo >&2 "Exec of tini failed!" + exit 1 + ;; *) # Default case if the program selector does not match echo "Unknown program: $program_selector" diff --git a/lotman/lotman.go b/lotman/lotman.go index ee3544ffd..69ac4d302 100644 --- a/lotman/lotman.go +++ b/lotman/lotman.go @@ -1,4 +1,4 @@ -//go:build windows || darwin || linux +//go:build !lotman || (lotman && linux && ppc64le) || !linux // For now we're shutting off LotMan due to weirdness with purego. When we return to this, remember // that purego doesn't support (linux && ppc64le), so we'll need to add that back here. diff --git a/lotman/lotman_linux.go b/lotman/lotman_linux.go index 910859f90..1719911e3 100644 --- a/lotman/lotman_linux.go +++ b/lotman/lotman_linux.go @@ -1,5 +1,4 @@ -//go:build false -// For now we're shutting off LotMan due to weirdness with purego +//go:build lotman && linux && !ppc64le /*************************************************************** * @@ -25,6 +24,7 @@ package lotman import ( "bytes" + "context" "encoding/json" "fmt" "os" @@ -36,6 +36,7 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/param" ) @@ -58,6 +59,7 @@ var ( // Auxilliary functions LotmanLotExists func(lotName string, errMsg *[]byte) int32 LotmanSetContextStr func(contextKey string, contextValue string, errMsg *[]byte) int32 + LotmanGetContextStr func(key string, output *[]byte, errMsg *[]byte) int32 // Functions that would normally take a char *** as an argument take an *unsafe.Pointer instead because // these functions are responsible for allocating and deallocating the memory for the char ***. The Go // runtime will handle the memory management for the *unsafe.Pointer. @@ -299,13 +301,17 @@ func GetAuthorizedCallers(lotName string) (*[]string, error) { // 1. The federation's discovery url // 2. The federation's director url // TODO: Consider what happens to the lot if either of these values change in the future after the lot is created? -func getFederationIssuer() string { - federationIssuer := param.Federation_DiscoveryUrl.GetString() +func getFederationIssuer() (string, error) { + fedInfo, err := config.GetFederation(context.Background()) + if err != nil { + return "", err + } + federationIssuer := fedInfo.DiscoveryEndpoint if federationIssuer == "" { - federationIssuer = param.Federation_DirectorUrl.GetString() + federationIssuer = fedInfo.DirectorEndpoint } - return federationIssuer + return federationIssuer, nil } // Initialize the LotMan library and bind its functions to the global vars @@ -334,6 +340,7 @@ func InitLotman() bool { // Auxilliary functions purego.RegisterLibFunc(&LotmanLotExists, lotmanLib, "lotman_lot_exists") purego.RegisterLibFunc(&LotmanSetContextStr, lotmanLib, "lotman_set_context_str") + purego.RegisterLibFunc(&LotmanGetContextStr, lotmanLib, "lotman_get_context_str") purego.RegisterLibFunc(&LotmanGetLotOwners, lotmanLib, "lotman_get_owners") purego.RegisterLibFunc(&LotmanGetLotParents, lotmanLib, "lotman_get_parent_names") purego.RegisterLibFunc(&LotmanGetLotsFromDir, lotmanLib, "lotman_get_lots_from_dir") @@ -359,14 +366,22 @@ func InitLotman() bool { log.Warningf("Error while unmarshaling Lots from config: %v", err) } - federationIssuer := getFederationIssuer() + federationIssuer, err := getFederationIssuer() + if err != nil { + log.Errorf("Error getting federation issuer: %v", err) + return false + } + if federationIssuer == "" { + log.Errorln("Unable to determine federation issuer which is needed by Lotman to determine lot ownership") + return false + } callerMutex.Lock() defer callerMutex.Unlock() ret = LotmanSetContextStr("caller", federationIssuer, &errMsg) if ret != 0 { trimBuf(&errMsg) - log.Errorf("Error setting context for default lot: %s", string(errMsg)) + log.Errorf("Error setting caller context to %s for default lot: %s", federationIssuer, string(errMsg)) return false } @@ -527,6 +542,7 @@ func InitLotman() bool { if ret != 0 { trimBuf(&errMsg) log.Errorf("Error creating lot %s: %s", lot.LotName, string(errMsg)) + log.Infoln("Full lot JSON passed to Lotman for lot creation:", string(lotJSON)) return false } } diff --git a/lotman/lotman_test.go b/lotman/lotman_test.go index 61975f6b1..b5a12c504 100644 --- a/lotman/lotman_test.go +++ b/lotman/lotman_test.go @@ -1,6 +1,5 @@ -//go:build false +//go:build lotman && linux && !ppc64le -//linux && !ppc64le /*************************************************************** * * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research @@ -26,6 +25,8 @@ import ( _ "embed" "encoding/json" "fmt" + "net/http" + "net/http/httptest" "os" "strings" "testing" @@ -37,19 +38,33 @@ import ( ) //go:embed resources/lots-config.yaml - var yamlMockup string -func setupLotmanFromConf(t *testing.T, readConfig bool, name string) (bool, func()) { - // Load in our config - if readConfig { - viper.Set("Federation.DiscoveryUrl", "https://fake-federation.com") - viper.SetConfigType("yaml") - err := viper.ReadConfig(strings.NewReader(yamlMockup)) - if err != nil { - t.Fatalf("Error reading config: %v", err) - } - } +// Initialize Lotman +// If we read from the embedded yaml, we need to override the SHOULD_OVERRIDE keys with the discUrl +// so that underlying metadata discovery can happen against the mock discovery host +func setupLotmanFromConf(t *testing.T, readConfig bool, name string, discUrl string) (bool, func()) { + // Load in our config and handle overriding the SHOULD_OVERRIDE keys with the discUrl + // Load in our config + if readConfig { + viper.SetConfigType("yaml") + err := viper.ReadConfig(strings.NewReader(yamlMockup)) + if err != nil { + t.Fatalf("Error reading config: %v", err) + } + + // Traverse the settings and modify the "Owner" keys directly in Viper + lots := viper.Get("Lotman.Lots").([]interface{}) + for i, lot := range lots { + if lotMap, ok := lot.(map[string]interface{}); ok { + if owner, ok := lotMap["owner"].(string); ok && owner == "SHOULD_OVERRIDE" { + lotMap["owner"] = discUrl + lots[i] = lotMap + } + } + } + viper.Set("Lotman.Lots", lots) + } tmpPathPattern := name + "*" tmpPath, err := os.MkdirTemp("", tmpPathPattern) @@ -63,21 +78,43 @@ func setupLotmanFromConf(t *testing.T, readConfig bool, name string) (bool, func } } +// Create a mock discovery host that returns the servers URL as the value for each pelican-configuration key +func getMockDiscoveryHost() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/pelican-configuration" { + w.Header().Set("Content-Type", "application/json") + serverURL := r.Host + response := fmt.Sprintf(`{ + "director_endpoint": "https://%s/osdf-director.osg-htc.org", + "namespace_registration_endpoint": "https://%s/osdf-registry.osg-htc.org", + "jwks_uri": "https://%s/osdf/public_signing_key.jwks" +}`, serverURL, serverURL, serverURL) + w.Write([]byte(response)) + } else { + http.NotFound(w, r) + } + })) +} + // Test the library initializer. NOTE: this also tests CreateLot, which is a part of initialization. func TestLotmanInit(t *testing.T) { server_utils.ResetTestState() t.Run("TestBadInit", func(t *testing.T) { // We haven't set various bits needed to create the lots, like discovery URL - success, cleanup := setupLotmanFromConf(t, false, "LotmanBadInit") + success, cleanup := setupLotmanFromConf(t, false, "LotmanBadInit", "") defer cleanup() require.False(t, success) }) t.Run("TestGoodInit", func(t *testing.T) { viper.Set("Log.Level", "debug") - viper.Set("Federation.DiscoveryUrl", "https://fake-federation.com") - success, cleanup := setupLotmanFromConf(t, false, "LotmanGoodInit") + server := getMockDiscoveryHost() + // Set the Federation.DiscoveryUrl to the test server's URL + // Lotman uses the discovered URLs/keys to determine some aspects of lot ownership + viper.Set("Federation.DiscoveryUrl", server.URL) + + success, cleanup := setupLotmanFromConf(t, false, "LotmanGoodInit", server.URL) defer cleanup() require.True(t, success) @@ -95,7 +132,7 @@ func TestLotmanInit(t *testing.T) { err := json.Unmarshal(defaultOutput, &defaultLot) require.NoError(t, err, fmt.Sprintf("Error unmarshalling default lot JSON: %s", string(defaultOutput))) require.Equal(t, "default", defaultLot.LotName) - require.Equal(t, "https://fake-federation.com", defaultLot.Owner) + require.Equal(t, server.URL, defaultLot.Owner) require.Equal(t, "default", defaultLot.Parents[0]) require.Equal(t, 0.0, *(defaultLot.MPA.DedicatedGB)) require.Equal(t, int64(0), (defaultLot.MPA.MaxNumObjects.Value)) @@ -111,7 +148,7 @@ func TestLotmanInit(t *testing.T) { err = json.Unmarshal(rootOutput, &rootLot) require.NoError(t, err, fmt.Sprintf("Error unmarshalling root lot JSON: %s", string(rootOutput))) require.Equal(t, "root", rootLot.LotName) - require.Equal(t, "https://fake-federation.com", rootLot.Owner) + require.Equal(t, server.URL, rootLot.Owner) require.Equal(t, "root", rootLot.Parents[0]) require.Equal(t, 0.0, *(rootLot.MPA.DedicatedGB)) require.Equal(t, int64(0), (rootLot.MPA.MaxNumObjects.Value)) @@ -120,8 +157,9 @@ func TestLotmanInit(t *testing.T) { func TestLotmanInitFromConfig(t *testing.T) { server_utils.ResetTestState() - - success, cleanup := setupLotmanFromConf(t, true, "LotmanInitConf") + server := getMockDiscoveryHost() + viper.Set("Federation.DiscoveryUrl", server.URL) + success, cleanup := setupLotmanFromConf(t, true, "LotmanInitConf", server.URL) defer cleanup() require.True(t, success) @@ -140,7 +178,7 @@ func TestLotmanInitFromConfig(t *testing.T) { err := json.Unmarshal(defaultOutput, &defaultLot) require.NoError(t, err, fmt.Sprintf("Error unmarshalling default lot JSON: %s", string(defaultOutput))) require.Equal(t, "default", defaultLot.LotName) - require.Equal(t, "https://fake-federation.com", defaultLot.Owner) + require.Equal(t, server.URL, defaultLot.Owner) require.Equal(t, "default", defaultLot.Parents[0]) require.Equal(t, 100.0, *(defaultLot.MPA.DedicatedGB)) require.Equal(t, int64(1000), (defaultLot.MPA.MaxNumObjects.Value)) @@ -157,7 +195,7 @@ func TestLotmanInitFromConfig(t *testing.T) { err = json.Unmarshal(rootOutput, &rootLot) require.NoError(t, err, fmt.Sprintf("Error unmarshalling root lot JSON: %s", string(rootOutput))) require.Equal(t, "root", rootLot.LotName) - require.Equal(t, "https://fake-federation.com", rootLot.Owner) + require.Equal(t, server.URL, rootLot.Owner) require.Equal(t, "root", rootLot.Parents[0]) require.Equal(t, 1.0, *(rootLot.MPA.DedicatedGB)) require.Equal(t, int64(10), rootLot.MPA.MaxNumObjects.Value) @@ -220,7 +258,9 @@ func TestGetLotmanLib(t *testing.T) { func TestGetAuthzCallers(t *testing.T) { server_utils.ResetTestState() - success, cleanup := setupLotmanFromConf(t, true, "LotmanGetAuthzCalleres") + server := getMockDiscoveryHost() + viper.Set("Federation.DiscoveryUrl", server.URL) + success, cleanup := setupLotmanFromConf(t, true, "LotmanGetAuthzCalleres", server.URL) defer cleanup() require.True(t, success) @@ -229,7 +269,7 @@ func TestGetAuthzCallers(t *testing.T) { authzedCallers, err := GetAuthorizedCallers("test-2") require.NoError(t, err, "Failed to get authorized callers") require.Equal(t, 2, len(*authzedCallers)) - require.Contains(t, *authzedCallers, "https://fake-federation.com") + require.Contains(t, *authzedCallers, server.URL) require.Contains(t, *authzedCallers, "https://different-fake-federation.com") // test with a non-existent lot @@ -239,7 +279,9 @@ func TestGetAuthzCallers(t *testing.T) { func TestGetLot(t *testing.T) { server_utils.ResetTestState() - success, cleanup := setupLotmanFromConf(t, true, "LotmanGetLot") + server := getMockDiscoveryHost() + viper.Set("Federation.DiscoveryUrl", server.URL) + success, cleanup := setupLotmanFromConf(t, true, "LotmanGetLot", server.URL) defer cleanup() require.True(t, success) @@ -251,7 +293,7 @@ func TestGetLot(t *testing.T) { require.Contains(t, lot.Parents, "root") require.Contains(t, lot.Parents, "test-1") require.Equal(t, 3, len(lot.Owners)) - require.Contains(t, lot.Owners, "https://fake-federation.com") + require.Contains(t, lot.Owners, server.URL) require.Contains(t, lot.Owners, "https://different-fake-federation.com") require.Contains(t, lot.Owners, "https://another-fake-federation.com") require.Equal(t, 1.11, *(lot.MPA.DedicatedGB)) @@ -262,7 +304,9 @@ func TestGetLot(t *testing.T) { func TestUpdateLot(t *testing.T) { server_utils.ResetTestState() - success, cleanup := setupLotmanFromConf(t, true, "LotmanInitConf") + server := getMockDiscoveryHost() + viper.Set("Federation.DiscoveryUrl", server.URL) + success, cleanup := setupLotmanFromConf(t, true, "LotmanInitConf", server.URL) defer cleanup() require.True(t, success) @@ -285,7 +329,7 @@ func TestUpdateLot(t *testing.T) { }, } - err := UpdateLot(&lotUpdate, "https://fake-federation.com") + err := UpdateLot(&lotUpdate, server.URL) require.NoError(t, err, "Failed to update lot") // Now check that the update was successful @@ -300,12 +344,14 @@ func TestUpdateLot(t *testing.T) { func TestDeleteLotsRec(t *testing.T) { server_utils.ResetTestState() - success, cleanup := setupLotmanFromConf(t, true, "LotmanInitConf") + server := getMockDiscoveryHost() + viper.Set("Federation.DiscoveryUrl", server.URL) + success, cleanup := setupLotmanFromConf(t, true, "LotmanInitConf", server.URL) defer cleanup() require.True(t, success) // Delete test-1, then verify both it and test-2 are gone - err := DeleteLotsRecursive("test-1", "https://fake-federation.com") + err := DeleteLotsRecursive("test-1", server.URL) require.NoError(t, err, "Failed to delete lot") // Now check that the delete was successful diff --git a/lotman/lotman_ui.go b/lotman/lotman_ui.go index 57d44a856..b5dad97b8 100644 --- a/lotman/lotman_ui.go +++ b/lotman/lotman_ui.go @@ -1,5 +1,4 @@ -//go:build false -//linux && !ppc64le +//go:build lotman && linux && !ppc64le /*************************************************************** * @@ -37,7 +36,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/pelicanplatform/pelican/config" - "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" "github.com/pelicanplatform/pelican/server_utils" "github.com/pelicanplatform/pelican/token" @@ -118,7 +116,10 @@ func VerifyNewLotToken(lot *Lot, strToken string) (bool, error) { if len(lot.Parents) != 0 && lot.Parents[0] == "root" { // We check that the token is signed by the federation // First check for discovery URL and then for director URL, both of which should host the federation's pubkey - issuerUrl := getFederationIssuer() + issuerUrl, err := getFederationIssuer() + if err != nil { + return false, err + } kSet, err := server_utils.GetJWKSFromIssUrl(issuerUrl) if err != nil { @@ -201,7 +202,12 @@ func VerifyNewLotToken(lot *Lot, strToken string) (bool, error) { // Get the namespace by querying the director and checking the headers errMsgPrefix := "the provided token is acceptible, but no owner could be determined because " - directorUrlStr := param.Federation_DirectorUrl.GetString() + + fedInfo, err := config.GetFederation(context.Background()) + if err != nil { + return false, errors.Wrap(err, errMsgPrefix+"the federation information could not be retrieved") + } + directorUrlStr := fedInfo.DirectorEndpoint if directorUrlStr == "" { return false, errors.New(errMsgPrefix + "the federation director URL is not set") } @@ -212,7 +218,7 @@ func VerifyNewLotToken(lot *Lot, strToken string) (bool, error) { directorUrl.Path, err = url.JoinPath("/api/v1.0/director/object", path) if err != nil { - return false, errors.Wrap(err, errMsgPrefix+"the director URL could not be joined with the path") + return false, errors.Wrap(err, errMsgPrefix+"the director's object path could not be constructed") } // Get the namespace by querying the director and checking the headers. The client should NOT diff --git a/lotman/resources/lots-config.yaml b/lotman/resources/lots-config.yaml index 874b0b7a3..9fdf31edb 100644 --- a/lotman/resources/lots-config.yaml +++ b/lotman/resources/lots-config.yaml @@ -20,7 +20,7 @@ Lotman: Lots: - LotName: "default" - Owner: "https://fake-federation.com" + Owner: "SHOULD_OVERRIDE" Parents: - "default" ManagementPolicyAttrs: @@ -38,7 +38,7 @@ Lotman: Value: 123456 - LotName: "root" - Owner: "https://fake-federation.com" + Owner: "SHOULD_OVERRIDE" Parents: - "root" Paths: