diff --git a/Makefile b/Makefile index 07194a72..cd91142b 100644 --- a/Makefile +++ b/Makefile @@ -10,11 +10,12 @@ REPOS ?= tyk tyk-analytics tyk-pump tyk-identity-broker tyk-sink portal t GITHUB_TOKEN ?= $(shell pass me/github) JIRA_USER ?= alok@tyk.io JIRA_TOKEN ?= $(shell pass Tyk/atlassian) -VARIATION ?= prod-variations +VARIATION ?= prod-variation PC_TOKEN ?= $(shell pass Tyk/packagecloud) UNSTABLE_REPOS := tyk-gateway-unstable tyk-dashboard-unstable tyk-pump-unstable tyk-mdcb-unstable portal-unstable tyk-identity-broker-unstable tyk-sync-unstable STABLE_REPOS := tyk-gateway tyk-dashboard tyk-pump tyk-mdcb portal tyk-identity-broker + gromit: go.mod go.sum *.go $(SRC) $(TEMPLATES) update-variation go build -v -trimpath -ldflags "-X github.com/TykTechnologies/gromit/util.version=$(VERSION) -X github.com/TykTechnologies/gromit/util.commit=$(COMMIT) -X github.com/TykTechnologies/gromit/util.buildDate=$(BUILD_DATE)" go mod tidy diff --git a/cmd/policy.go b/cmd/policy.go index 975706dc..3e9851da 100644 --- a/cmd/policy.go +++ b/cmd/policy.go @@ -19,6 +19,7 @@ import ( "bytes" "fmt" "os" + "strings" "github.com/TykTechnologies/gromit/policy" "github.com/rs/zerolog/log" @@ -223,7 +224,8 @@ var matchSubCmd = &cobra.Command{ return err } - repos := []string{"tyk-ee", "tyk-analytics", "tyk-pump", "tyk-sink"} + rs, _ := cmd.Flags().GetString("repos") + repos := strings.Split(rs, ",") tagOverride := args[0] tagMatch := args[1] @@ -275,6 +277,7 @@ func init() { serveSubCmd.Flags().String("save", "testdata/tui", "Test variations are loaded from and saved to this directory") matchSubCmd.Flags().String("config", "$HOME/.docker/config.json", "Config file to read authentication token from") + matchSubCmd.Flags().String("repos", "tyk-ee,tyk-analytics,tyk-pump,tyk-sink", "Config file to read authentication token from") policyCmd.AddCommand(matchSubCmd) policyCmd.AddCommand(syncSubCmd) diff --git a/config/config.yaml b/config/config.yaml index cd14a551..e00ff8b6 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,6 +1,6 @@ # `policy` is the root object for the policy and bundle commands. Any # parent policy options provided here can be overridden by repo -# specific/ branch specific options. +# specific/branch specific options. policy: # Owner component of Github reponame @@ -43,7 +43,7 @@ policy: buildenv: 1.23-bullseye # baseimage what the container images are based on. It needs to be a # debian compatible distro as the Dockerfile assumes apt-get - baseimage: debian:bookworm-slim + baseimage: debian:trixie-slim # distrolessbaseimage is used for container images when the # distroless feature is set. base is appropriate for cgo # builds, static for pure go. @@ -53,20 +53,62 @@ policy: # Repo specific policy for `tyk` starts here-> tyk: distrolessbaseimage: base-debian12:latest - # Description for the repo - this will be used as description for - # deb/rpm packages, docker images etc. - description: >- - Tyk Open Source API Gateway written in Go, supporting REST, GraphQL, TCP and gRPC protocols - # The packagecloud repo name for this git repo. - pcrepo: tyk-gateway - # The dockerhub image name for this repo. in the form of repo/image - dhrepo: tykio/tyk-gateway - # Cloudsmith image name for this repo. - csrepo: docker.tyk.io/tyk-gateway/tyk-gateway - # (eg: TykTechnologies/devops) that will be automatically set as - # It also is used while generating dependabot templates to set - # The name for the deb/rpm packages generated by release workflow. + # packagename is the historical name of the package. Builds + # may produce artifacts named differently but the + # directories and contents of the package use this name. packagename: tyk-gateway + # Builds and corresponding packages. There is no static + # compatibility check, the only way to know if the + # combination of cgo, flags and archs work is to look at the + # test results in the repo + builds: + # std builds are open access + std: + # passed to go build -flags + flags: + - -tags=goplugin + # Description for the repo - this will be used as description for + # deb/rpm packages, docker images etc. + description: >- + Tyk Open Source API Gateway written in Go, supporting REST, GraphQL, TCP and gRPC protocols + # imagetitle goes into the OCI labels + imagetitle: Tyk Gateway + # The name for the deb/rpm packages generated by release workflow. + buildpackagename: tyk-gateway + # The packagecloud repo to test upgrades from + upgraderepo: tyk-gateway + # The packagecloud repo for package uploads + pcrepo: tyk-gateway-unstable + # The dockerhub image name for this repo. in the form of repo/image + dhrepo: tykio/tyk-gateway + # Cloudsmith image name for this repo. + csrepo: docker.tyk.io/tyk-gateway/tyk-gateway + # CI repo name, regsitry is inferred in release.yml + cirepo: tyk + # List of target architectures for cross compilation + archs: + - go: amd64 + deb: amd64 + docker: linux/amd64 + - go: arm64 + deb: aarch64 + docker: linux/arm64 + - go: s390x + deb: s390x + docker: linux/s390x + fips: + flags: + - -tags=goplugin,fips,boringcrypto + buildpackagename: tyk-gateway-fips + description: >- + Tyk Open Source API Gateway written in Go, supporting REST, GraphQL, TCP and gRPC protocols + Built with boringssl + archs: + - go: amd64 + deb: amd64 + docker: linux/amd64 + env: + - GOEXPERIMENT=boringcrypto # The name of the binary generated after the build. binary: tyk # `protected` specifies repo specific protected branches - this gets @@ -97,38 +139,78 @@ policy: # Branch specific policy for different branches starts here.-> branches: master: + builds: + std: + flags: + - -trimpath + # ee builds are enterprise + ee: + flags: + - -tags=goplugin,ee + description: >- + Tyk API Gateway Enterprise Edition written in Go, supporting REST, GraphQL, TCP and gRPC protocols + imagetitle: Tyk Gateway Enterprise Edition + buildpackagename: tyk-gateway-ee + pcrepo: tyk-ee-unstable + dhrepo: tykio/tyk-gateway-ee + cirepo: tyk-ee + archs: + - go: amd64 + deb: amd64 + docker: linux/amd64 + - go: arm64 + deb: aarch64 + docker: linux/arm64 + - go: s390x + deb: s390x + docker: linux/s390x features: - - s390x - release-test - distroless - - ee - - fips - release-5.7: + release-5.3: buildenv: 1.22-bullseye features: - - s390x - - release-test - distroless - - ee - - fips + - release-test + release-5-lts: + buildenv: 1.16 + features: + - plugin-compiler-fix-vendor + - python-support release-5.8: - buildenv: 1.23-bullseye + builds: + ee: + flags: + - -tags=goplugin,ee + description: >- + Tyk API Gateway Enterprise Edition written in Go, supporting REST, GraphQL, TCP and gRPC protocols + imagetitle: Tyk Gateway Enterprise Edition + buildpackagename: tyk-gateway-ee + pcrepo: tyk-ee-unstable + dhrepo: tykio/tyk-gateway-ee + cirepo: tyk-ee + archs: + - go: amd64 + deb: amd64 + docker: linux/amd64 + - go: arm64 + deb: aarch64 + docker: linux/arm64 + - go: s390x + deb: s390x + docker: linux/s390x features: - - s390x - release-test - distroless - - ee - - fips tyk-analytics: - description: >- - Dashboard for the Tyk API Gateway - pcrepo: tyk-dashboard + pcrepo: tyk-dashboard-unstable dhrepo: tykio/tyk-dashboard csrepo: docker.tyk.io/tyk-dashboard/tyk-dashboard exposeports: "3000 5000" - binary: tyk-analytics packagename: tyk-dashboard + binary: tyk-analytics + buildpackagename: tyk-dashboard cgo: true upgradefromver: 3.0.9 configfile: tyk_analytics.conf @@ -139,20 +221,66 @@ policy: tests: - ui - api + builds: + std: + description: >- + Dashboard for the Tyk API Gateway + imagetitle: Tyk Dashboard + buildpackagename: tyk-dashboard + pcrepo: tyk-dashboard-unstable + upgraderepo: tyk-dashboard + dhrepo: tykio/tyk-dashboard + csrepo: docker.tyk.io/tyk-dashboard/tyk-dashboard + cirepo: tyk-analytics + archs: + - go: amd64 + deb: amd64 + docker: linux/amd64 + - go: arm64 + deb: aarch64 + docker: linux/arm64 + - go: s390x + deb: s390x + docker: linux/s390x + fips: + flags: + - -tags=fips,boringcrypto + description: >- + Dashboard for the Tyk API Gateway. + This version is compiled with boringssl. + buildpackagename: tyk-pump-fips + archs: + - go: amd64 + deb: amd64 + docker: linux/amd64 + env: + - GOEXPERIMENT=boringcrypto + ee: + flags: + - -tags=goplugin,ee + description: >- + Enterprise Edition Dashboard for the Tyk API Gateway. + imagetitle: Tyk Dashboard Enterprise Edition + buildpackagename: tyk-dashboard-ee + pcrepo: tyk-ee-unstable + dhrepo: tykio/tyk-gateway-ee + cirepo: tyk-ee + archs: + - go: amd64 + deb: amd64 + docker: linux/amd64 + - go: arm64 + deb: aarch64 + docker: linux/arm64 + - go: s390x + deb: s390x + docker: linux/s390x branches: master: cgo: false features: - nightly-e2e - distroless - - s390x - - distroless - - release-test - release-5.7: - buildenv: 1.22-bullseye - features: - - distroless - - s390x - release-test release-5.8: buildenv: 1.23-bullseye @@ -160,18 +288,13 @@ policy: features: - nightly-e2e - distroless - - s390x - - distroless - release-test portal: - description: >- - Developer portal for the Tyk API Gateway - pcrepo: portal - dhrepo: tykio/portal exposeports: 80 - binary: dev-portal packagename: portal + binary: dev-portal + buildpackagename: portal buildenv: 1.22-bullseye cgo: true upgradefromver: 1.0.0 @@ -179,6 +302,25 @@ policy: versionpackage: github.com/TykTechnologies/portal/model/version features: - distroless + builds: + std: + buildpackagename: portal + description: >- + Developer portal for the Tyk API Gateway + pcrepo: portal-unstable + upgraderepo: portal + dhrepo: tykio/portal + cirepo: portal + archs: + - go: amd64 + deb: amd64 + docker: linux/amd64 + - go: arm64 + deb: aarch64 + docker: linux/arm64 + - go: s390x + deb: s390x + docker: linux/s390x branches: master: @@ -191,31 +333,52 @@ policy: distrolessbaseimage: static-debian12:nonroot repos: tyk-pump: - description: >- - Tyk Analytics Pump to move analytics data from Redis to any supported back end (multiple back ends can be written to at once). - pcrepo: tyk-pump - dhrepo: tykio/tyk-pump-docker-pub - csrepo: docker.tyk.io/tyk-pump/tyk-pump + packagename: tyk-pump exposeports: "80" binary: tyk-pump - packagename: tyk-pump cgo: false upgradefromver: 1.6.0 configfile: pump.conf versionpackage: github.com/TykTechnologies/tyk-pump/pumps tests: [api] - features: - - s390x - - distroless + builds: + std: + description: >- + Tyk Analytics Pump to move analytics data from Redis to any supported back end (multiple back ends can be written to at once). + imagetitle: Tyk Analytics Pump + buildpackagename: tyk-pump + upgraderepo: tyk-pump + pcrepo: tyk-pump-unstable + dhrepo: tykio/tyk-pump-docker-pub + csrepo: docker.tyk.io/tyk-pump/tyk-pump + cirepo: tyk-pump + archs: + - go: amd64 + deb: amd64 + docker: linux/amd64 + - go: arm64 + deb: aarch64 + docker: linux/arm64 + - go: s390x + deb: s390x + docker: linux/s390x + fips: + flags: + - -tags=fips,boringcrypto + buildpackagename: tyk-pump-fips + description: >- + Tyk Analytics Pump to move analytics data from Redis to any supported back end (multiple back ends can be written to at once). + This version is compiled with boringssl. + archs: + - go: amd64 + deb: amd64 + docker: linux/amd64 + env: + - GOEXPERIMENT=boringcrypto branches: master: tyk-identity-broker: - description: >- - Tyk Authentication Proxy for third-party login - pcrepo: tyk-identity-broker - dhrepo: tykio/tyk-identity-broker - csrepo: docker.tyk.io/tyk-identity-broker/tyk-identity-broker exposeports: 80 binary: tyk-identity-broker packagename: tyk-identity-broker @@ -223,47 +386,90 @@ policy: upgradefromver: 1.1.0 configfile: tib.conf versionpackage: github.com/TykTechnologies/tyk-identity-broker/main + builds: + std: + description: >- + Tyk Authentication Proxy for third-party login + imagetitle: Tyk Identity Broker + buildpackagename: tyk-identity-broker + pcrepo: tyk-identity-broker-unstable + upgraderepo: tyk-identity-broker + dhrepo: tykio/tyk-identity-broker + csrepo: docker.tyk.io/tyk-identity-broker/tyk-identity-broker + cirepo: tyk-identity-broker + archs: + - go: amd64 + deb: amd64 + docker: linux/amd64 + - go: arm64 + deb: aarch64 + docker: linux/arm64 + - go: s390x + deb: s390x + docker: linux/s390x branches: master: tyk-sink: - description: >- - Tyk RPC server backend (bridge) - pcrepo: tyk-mdcb - dhrepo: tykio/tyk-mdcb-docker - csrepo: docker.tyk.io/tyk-sink/tyk-sink buildenv: 1.22-bullseye exposeports: 80 binary: tyk-sink - packagename: tyk-sink cgo: false + packagename: tyk-sink upgradefromver: 1.8.2 configfile: tyk_sink.conf versionpackage: github.com/TykTechnologies/tyk-sink/main tests: [api] - features: - - s390x - - distroless + builds: + std: + description: >- + Tyk RPC server backend (bridge) + imagetitle: Tyk Dashboard + buildpackagename: tyk-sink + pcrepo: tyk-mdcb-unstable + upgraderepo: tyk-mdcb + dhrepo: tykio/tyk-mdcb-docker + cirepo: tyk-sink + archs: + - go: amd64 + deb: amd64 + docker: linux/amd64 + - go: arm64 + deb: aarch64 + docker: linux/arm64 + - go: s390x + deb: s390x + docker: linux/s390x branches: master: midsommar: - description: >- - Tyk AI Portal - pcrepo: midsommar - dhrepo: tykio/midsommar - csrepo: docker.tyk.io/midsommar/midsommar buildenv: 1.22-bullseye exposeports: 80 - binary: midsommar packagename: midsommar + binary: midsommar cgo: false upgradefromver: 0.0.1 configfile: midsommar.conf versionpackage: github.com/TykTechnologies/midsommar/main - features: - - s390x - - distroless + builds: + std: + buildpackagename: midsommar + description: >- + Tyk AI Portal + pcrepo: midsommar + dhrepo: tykio/midsommar + cirepo: midsommar + archs: + - go: amd64 + deb: amd64 + docker: linux/amd64 + - go: arm64 + deb: aarch64 + docker: linux/arm64 + - go: s390x + deb: s390x + docker: linux/s390x branches: main: @@ -272,7 +478,7 @@ policy: test-square: repos: tyk-pro: - owner: tyklabs + owner: TykTechnologies deletedfiles: - .github/workflows/release-tests.yml description: >- @@ -286,7 +492,7 @@ policy: main: # pkgs is used by the pkgs subcommand to weed packagecloud. Unlike the -# policy, key, you cannot override parameters from other levels. +# policy key, you cannot override parameters from other levels. # Precedence is exceptions > versioncutoff > agecutoff # Quirks of x/mod/semver: diff --git a/go.mod b/go.mod index e9e8b6d7..7ba79806 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/TykTechnologies/gromit go 1.24 require ( + dario.cat/mergo v1.0.1 github.com/Masterminds/sprig/v3 v3.3.0 github.com/ProtonMail/go-crypto v1.1.6 github.com/aws/aws-sdk-go-v2 v1.36.3 @@ -44,7 +45,6 @@ require ( ) require ( - dario.cat/mergo v1.0.1 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.3.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect diff --git a/policy/bundle_test.go b/policy/bundle_test.go index 14b26b82..ddb514bf 100644 --- a/policy/bundle_test.go +++ b/policy/bundle_test.go @@ -55,6 +55,7 @@ func TestBundleRender(t *testing.T) { if err != nil { t.Fatalf("Error creating temp dir: %v", err) } + t.Logf("templates rendered to: %s", tmpDir) defer os.RemoveAll(tmpDir) err = rp.SetBranch("master") diff --git a/policy/policy.go b/policy/policy.go index fd9a769c..501c1610 100644 --- a/policy/policy.go +++ b/policy/policy.go @@ -1,17 +1,20 @@ package policy import ( - "bytes" "fmt" "os" "path/filepath" + "slices" "time" + "maps" + "github.com/TykTechnologies/gromit/util" + + "dario.cat/mergo" "github.com/jinzhu/copier" "github.com/rs/zerolog/log" "github.com/spf13/viper" - "golang.org/x/exp/maps" ) // repoConfig contains all the attributes of a repo. Each element here @@ -21,15 +24,11 @@ import ( // levels type repoConfig struct { Owner string - Description string - PCRepo string - DHRepo string - CSRepo string - PackageName string - Reviewers []string ExposePorts string + PackageName string Binary string Buildenv string + Builds buildMap BaseImage string DistrolessBaseImage string Cgo bool @@ -43,6 +42,27 @@ type repoConfig struct { Repos map[string]repoConfig `copier:"-"` } +// build models the variations in build and their corresponding packages +type build struct { + Flags []string + BuildPackageName string + Description string + ImageTitle string + PCRepo string + UpgradeRepo string + DHRepo string + CSRepo string + CIRepo string + Env []string + Archs []struct { + Docker string + Deb string + Go string + } +} + +type buildMap map[string]*build + // Policies models the config file structure. There are three levels // at which a particular value can be set: group-level, repo, branch. // The group level is applicable for all the repos in that group. @@ -67,6 +87,7 @@ type branchVals struct { UpgradeFromVer string Tests []string Features []string + Builds buildMap DeletedFiles []string } @@ -78,13 +99,10 @@ type branchVals struct { type RepoPolicy struct { Owner string Name string - Description string Default string - PCRepo string - DHRepo string - CSRepo string - Binary string PackageName string + Binary string + Builds buildMap Reviewers []string ExposePorts string Cgo bool @@ -94,9 +112,7 @@ type RepoPolicy struct { Branch string Branchvals branchVals Branches map[string]branchVals - prBranch string Timestamp string - Visibility string } // PushOptions collects the input required to update templates for a @@ -116,7 +132,6 @@ func (rp *RepoPolicy) SetTimestamp(ts time.Time) { ts = time.Now().UTC() } rp.Timestamp = ts.Format(time.UnixDate) - } // GetTimeStamp returns the timestamp currently set for the given repopolicy. @@ -127,7 +142,7 @@ func (rp *RepoPolicy) GetTimeStamp() (time.Time, error) { return ts, err } -// SetBranch sets the Branch and Branchvals properties so that templates can simply access them instead of looking them up in the Branches map +// SetBranch sets the Branch and Branchvals properties so that templates can simply access them instead of looking them up in the Branches map. This must be called before calling Render() func (rp *RepoPolicy) SetBranch(branch string) error { bv, found := rp.Branches[branch] if !found { @@ -141,7 +156,7 @@ func (rp *RepoPolicy) SetBranch(branch string) error { // GetAllBranches returns all the branches that are managed for this repo func (rp *RepoPolicy) GetAllBranches() []string { - return maps.Keys(rp.Branches) + return slices.Sorted(maps.Keys(rp.Branches)) } // GetRepoPolicy will fetch the RepoPolicy for the supplied repo with @@ -194,6 +209,9 @@ func (p *Policies) GetRepoPolicy(repo string) (RepoPolicy, error) { if err != nil { return rp, err } + // builds are merged + log.Debug().Msgf("Merging builds for %s/%s", rp.Name, b) + rbv.Builds = mergeBuilds(r.Builds, bbv.Builds) // attributes that are unions rbv.Features = util.NewSetFromSlices(group.Features, r.Features, bbv.Features).Members() rbv.DeletedFiles = util.NewSetFromSlices(p.DeletedFiles, group.DeletedFiles, r.DeletedFiles, bbv.DeletedFiles).Members() @@ -205,6 +223,16 @@ func (p *Policies) GetRepoPolicy(repo string) (RepoPolicy, error) { return rp, nil } +// mergeBuilds returns a merged build map from _r_epo and _b_ranch level +func mergeBuilds(r, b buildMap) buildMap { + merged := make(buildMap) + maps.Copy(merged, r) + if err := mergo.Merge(&merged, b, mergo.WithOverride, mergo.WithAppendSlice); err != nil { + log.Fatal().Interface("dst", merged).Interface("src", b).Msgf("could not merge branch-level build definitions for: %v", err) + } + return merged +} + // ProcessBranch will render the templates into a git worktree for the supplied branch, commit and push the changes upstream // The upstream branch name is the supplied branch name prefixed with releng/ and is returned func (rp *RepoPolicy) ProcessBranch(pushOpts *PushOptions) error { @@ -271,36 +299,6 @@ func (rp *RepoPolicy) ProcessBranch(pushOpts *PushOptions) error { return nil } -// Stringer implementation for Policies -func (p Policies) String() string { - w := new(bytes.Buffer) - for _, grp := range p.Groups { - for repo, crPol := range grp.Repos { - fmt.Fprintf(w, "%s: package %s, image %s", repo, crPol.PackageName, crPol.DHRepo) - rp, err := p.GetRepoPolicy(repo) - if err != nil { - log.Fatal().Str("repo", repo).Err(err).Msg("failed to get policy, this should not happen") - } - fmt.Fprintf(w, " %s\n", rp) - } - } - return w.String() -} - -// Stringer implementation for RepoPolicy -func (rp RepoPolicy) String() string { - w := new(bytes.Buffer) - for b, bv := range rp.Branches { - fmt.Fprintf(w, " %s: package %s, image %s, features %v", b, rp.PackageName, rp.DHRepo, bv.Features) - if len(bv.Buildenv) > 0 { - fmt.Fprintf(w, " built on %s", bv.Buildenv) - } else { - fmt.Fprintf(w, " not built") - } - } - return w.String() -} - // LoadRepoPolicies populates the supplied policies with the policy key from a the config file // This will panic if the type assertions fail func LoadRepoPolicies(policies *Policies) error { diff --git a/policy/policy_test.go b/policy/policy_test.go index 67771299..f8ee82df 100644 --- a/policy/policy_test.go +++ b/policy/policy_test.go @@ -29,7 +29,6 @@ func TestPolicyConfig(t *testing.T) { assert.EqualValues(t, "right", repo0.Branchvals.Buildenv, "testing branch-level override for main") assert.ElementsMatch(t, []string{"a", "b", "c", "d"}, repo0.Branchvals.Features, "testing merging of branchvals") assert.EqualValues(t, "repo0.conf", repo0.Branchvals.ConfigFile, "testing repo-level inheritance of branchvals") - assert.EqualValues(t, "Repo Zero", repo0.Description, "testing repo-level value") err = repo0.SetBranch("dev") if err != nil { @@ -37,13 +36,31 @@ func TestPolicyConfig(t *testing.T) { } assert.EqualValues(t, "stillright", repo0.Branchvals.Buildenv, "testing overrides for dev") assert.ElementsMatch(t, []string{"a", "b", "e", "f"}, repo0.Branchvals.Features, "testing merging") - assert.EqualValues(t, "Repo Zero", repo0.Description, "testing inheritance") repo1, err := pol.GetRepoPolicy("repo1") if err != nil { t.Fatalf("Could not get repo1: %v", err) } - assert.EqualValues(t, "Repo One", repo1.Description, "testing second repo") r1b := repo1.GetAllBranches() assert.EqualValues(t, []string{"main"}, r1b, "testing branches") + err = repo1.SetBranch("main") + if err != nil { + t.Fatalf("Could not set main branch: %v", err) + } + assert.EqualValues(t, []string{"flagstd1", "flagstd2"}, repo1.Branchvals.Builds["std"].Flags, "testing explicit merge") + assert.EqualValues(t, "repo1-std2", repo1.Branchvals.Builds["std2"].BuildPackageName, "testing implicit merges at branch") + assert.EqualValues(t, []string{"flag2"}, repo1.Branchvals.Builds["std2"].Flags, "testing implicit merge from repo") + assert.EqualValues(t, build{Flags: []string{"flagstd1", "flagstd2"}, + BuildPackageName: "repo1-pkg", + DHRepo: "repo1-doc-right", + Archs: []struct { + Docker string + Deb string + Go string + }{ + {"doc1", "deb1", "go1"}, + {"doc2", "deb2", "go2"}}, + }, *repo1.Branchvals.Builds["std"], "testing full merge") + assert.EqualValues(t, []string{"repo1-doc-right"}, repo1.GetImages("DHRepo"), "testing getImages()") + assert.EqualValues(t, []string{"doc1", "doc2"}, repo1.GetDockerPlatforms(), "testing getDockerPlatforms()") } diff --git a/policy/templates/distroless/ci/Dockerfile.distroless b/policy/templates/distroless/ci/Dockerfile.distroless index 4f6418c9..48b25342 100644 --- a/policy/templates/distroless/ci/Dockerfile.distroless +++ b/policy/templates/distroless/ci/Dockerfile.distroless @@ -1,17 +1,17 @@ # Generated by: gromit policy -FROM {{ .Branchvals.BaseImage }} as DEB +FROM {{ .Branchvals.BaseImage }} AS deb ARG TARGETARCH -ARG EDITION +ARG BUILD_PACKAGE_NAME ENV DEBIAN_FRONTEND=noninteractive -COPY *${TARGETARCH}.deb / -RUN rm -f /*fips*.deb && dpkg -i /{{ .PackageName }}${EDITION}_*${TARGETARCH}.deb && rm /*.deb +COPY ${BUILD_PACKAGE_NAME}*${TARGETARCH}.deb / +RUN dpkg -i /${BUILD_PACKAGE_NAME}_*${TARGETARCH}.deb && rm /*.deb FROM gcr.io/distroless/{{ .Branchvals.DistrolessBaseImage }} -COPY --from=DEB /opt/{{ .PackageName }} /opt/{{ .PackageName }} +COPY --from=deb /opt/{{ .PackageName }} /opt/{{ .PackageName }} ARG PORTS EXPOSE $PORTS diff --git a/policy/templates/el7-pgo-build/ci/goreleaser/goreleaser-el7.yml b/policy/templates/el7-pgo-build/ci/goreleaser/goreleaser-el7.yml index adefd57d..f2ae5dfc 100644 --- a/policy/templates/el7-pgo-build/ci/goreleaser/goreleaser-el7.yml +++ b/policy/templates/el7-pgo-build/ci/goreleaser/goreleaser-el7.yml @@ -4,18 +4,7 @@ # - arm64 # - amd64 -{{- if has "el7-pgo-build" .Branchvals.Features }} - {{- template "cgo_builds" . }} -{{- else }} - {{- template "builds" . }} -{{- end }} -{{- template "nfpm" . }} - -publishers: - - name: {{ .PCRepo }}-unstable - env: - - PACKAGECLOUD_TOKEN={{`{{ .Env.PACKAGECLOUD_TOKEN }}`}} - cmd: {{`packagecloud publish --debvers "{{ .Env.DEBVERS }}" --rpmvers "{{ .Env.RPMVERS }}"`}} tyk/{{ .PCRepo }}-unstable {{`{{ .ArtifactPath }}`}} +{{- template "builds" . }} # This disables archives archives: diff --git a/policy/templates/releng/.github/workflows/release.yml b/policy/templates/releng/.github/workflows/release.yml index e53865b9..5ede1986 100644 --- a/policy/templates/releng/.github/workflows/release.yml +++ b/policy/templates/releng/.github/workflows/release.yml @@ -11,11 +11,14 @@ name: Release # - devenv ECR # - Cloudsmith -# concurrency: -# group: {{`${{ github.workflow }}-${{ github.ref }}`}} -# cancel-in-progress: {{`${{ github.event_name == 'pull_request' }}`}} +concurrency: + group: {{`${{ github.workflow }}-${{ github.ref }}`}} + cancel-in-progress: {{`${{ github.event_name == 'pull_request' }}`}} on: + # Trigger release every monday at midnight for master CI images + schedule: + - cron: "0 0 * * 1" pull_request: push: branches: @@ -26,7 +29,7 @@ on: env: GOPRIVATE: github.com/TykTechnologies - VARIATION: prod-variations + VARIATION: prod-variation DOCKER_BUILD_SUMMARY: false DOCKER_BUILD_RECORD_UPLOAD: false # startsWith covers pull_request_target too diff --git a/policy/templates/releng/.github/workflows/release.yml.d/goreleaser.gotmpl b/policy/templates/releng/.github/workflows/release.yml.d/goreleaser.gotmpl index 4fa41bb3..8f2b354f 100644 --- a/policy/templates/releng/.github/workflows/release.yml.d/goreleaser.gotmpl +++ b/policy/templates/releng/.github/workflows/release.yml.d/goreleaser.gotmpl @@ -41,7 +41,10 @@ debvers: 'ubuntu/xenial ubuntu/bionic ubuntu/focal ubuntu/jammy debian/jessie debian/buster debian/bullseye debian/bookworm' {{- end }} {{/* el7-pgo-build */}} outputs: - tags: {{`${{ steps.ci_metadata.outputs.tags }}`}} +{{ $r := . }} +{{- range $b, $bv := $r.GetDockerBuilds }} + {{ $b }}_tags: {{printf "%s_%s%s" `${{ steps.ci_metadata` $b `.outputs.tags }}`}} +{{- end }} commit_author: {{`${{ steps.set_outputs.outputs.commit_author}}`}} steps: @@ -75,7 +78,6 @@ username: {{`${{ secrets.DOCKER_USERNAME }}`}} password: {{`${{ secrets.DOCKER_PASSWORD }}`}} - {{- if .CSRepo }} - name: Login to Cloudsmith if: startsWith(github.ref, 'refs/tags') uses: docker/login-action@v3 @@ -83,7 +85,6 @@ registry: docker.tyk.io username: {{`${{ secrets.CLOUDSMITH_USERNAME }}`}} password: {{`${{ secrets.CLOUDSMITH_API_KEY }}`}} - {{- end }} - uses: actions/cache@v4 with: @@ -159,17 +160,14 @@ with: mask-password: 'true' - {{ $editions := list "" }} - {{ if has "ee" .Branchvals.Features }} - {{ $editions = append $editions "ee" }} - {{ end }} - {{ range $edition := $editions }} - - name: Docker metadata for CI {{ $edition }} - id: ci_metadata - if: {{`${{ matrix.golang_cross == '` }}{{$.Branchvals.Buildenv}}{{`' }}`}} +{{- range $b, $bv := $r.GetDockerBuilds }} + - name: Docker metadata for {{ $b }} CI + id: ci_metadata_{{ $b }} + if: {{`${{ matrix.golang_cross == '` }}{{$r.Branchvals.Buildenv}}{{`' }}`}} uses: docker/metadata-action@v5 with: - images: {{`${{ steps.ecr.outputs.registry }}`}}/{{ $.Name }}{{ if eq $edition "ee" }}-ee{{ end }} + images: | + {{printf "%s/%s" `${{ steps.ecr.outputs.registry }}` $bv.CIRepo}} flavor: | latest=false tags: | @@ -180,13 +178,13 @@ type=semver,pattern={{`{{major}}.{{minor}}`}},prefix=v type=semver,pattern={{`{{version}}`}},prefix=v - - name: push image to CI {{ $edition }} - if: {{`${{ matrix.golang_cross == '` }}{{$.Branchvals.Buildenv}}{{`' }}`}} {{/* push only main build variation */}} + - name: push {{ $b }} image to CI + if: {{`${{ matrix.golang_cross == '` }}{{$r.Branchvals.Buildenv}}{{`' }}`}} {{/* push only main build variation */}} uses: docker/build-push-action@v6 with: context: "dist" - platforms: linux/amd64,linux/arm64 - {{- if has "distroless" $.Branchvals.Features }} + platforms: {{ $r.GetDockerPlatforms | join "," }} + {{- if has "distroless" $r.Branchvals.Features }} file: ci/Dockerfile.distroless {{- else }} file: ci/Dockerfile.std @@ -196,24 +194,19 @@ push: true cache-from: type=gha cache-to: type=gha,mode=max - tags: {{`${{ steps.ci_metadata` }}{{ $edition }}{{`.outputs.tags }}`}} - labels: {{`${{ steps.tag_metadata.outputs.labels }}`}} + tags: {{printf "%s_%s%s" `${{ steps.ci_metadata` $b `.outputs.tags }}`}} + labels: {{printf "%s_%s%s" `${{ steps.ci_metadata` $b `.outputs.labels }}`}} build-args: | - EDITION={{ if $edition }}-{{ end }}{{ $edition }} - {{ end }} + BUILD_PACKAGE_NAME={{ $bv.BuildPackageName }} - {{ $editions := list "" }} - {{ if has "ee" .Branchvals.Features }} - {{ $editions = append $editions "ee" }} - {{ end }} - {{ range $edition := $editions }} - - name: Docker metadata for tag push {{ $edition }} - id: tag_metadata + - name: Docker metadata for tag push + id: tag_metadata_{{ $b }} uses: docker/metadata-action@v5 with: images: | - {{ if eq $edition "ee" }}{{ $.DHRepo }}-ee{{ else }}{{ $.DHRepo }}{{ end }} - {{ if ne $edition "ee" }}{{ $.CSRepo }}{{ end }} + {{- range $image := $r.GetImages "DHRepo" "CSRepo" }} + {{ $image }} + {{end}} flavor: | latest=false prefix=v @@ -221,18 +214,18 @@ type=semver,pattern={{`{{major}}.{{minor}}`}} type=semver,pattern={{`{{version}}`}} labels: | - org.opencontainers.image.title={{ $.PackageName }}{{ if eq $edition "ee" }} Enterprise Edition{{ end }} {{ if has "distroless" $.Branchvals.Features }} (distroless) {{ end }} - org.opencontainers.image.description={{ $.Description }} + org.opencontainers.image.title={{ $bv.ImageTitle }} + org.opencontainers.image.description={{ $bv.Description }} org.opencontainers.image.vendor=tyk.io org.opencontainers.image.version={{`${{ github.ref_name }}`}} - - name: push image to prod {{ $edition }} - if: {{`${{ matrix.golang_cross == '` }}{{$.Branchvals.Buildenv}}{{`' }}`}} {{/* push only main build variation */}} + - name: push {{ $b }} image to prod + if: {{`${{ matrix.golang_cross == '` }}{{$r.Branchvals.Buildenv}}{{`' }}`}} {{/* push only main build variation */}} uses: docker/build-push-action@v6 with: context: "dist" - platforms: linux/amd64,linux/arm64 - {{- if has "distroless" $.Branchvals.Features }} + platforms: {{ $r.GetDockerPlatforms | join "," }} + {{- if has "distroless" $r.Branchvals.Features }} file: ci/Dockerfile.distroless {{- else }} file: ci/Dockerfile.std @@ -242,11 +235,11 @@ cache-from: type=gha cache-to: type=gha,mode=max push: {{`${{ startsWith(github.ref, 'refs/tags') }}`}} - tags: {{`${{ steps.tag_metadata.outputs.tags }}`}} - labels: {{`${{ steps.tag_metadata.outputs.labels }}`}} + tags: {{printf "%s_%s%s" `${{ steps.tag_metadata` $b `.outputs.tags }}`}} + labels: {{printf "%s_%s%s" `${{ steps.tag_metadata` $b `.outputs.labels }}`}} build-args: | - EDITION={{ if $edition }}-{{ end }}{{ $edition }} - {{ end }} + BUILD_PACKAGE_NAME={{ $bv.BuildPackageName }} +{{ end }} {{/* range getDockerBuilds */}} - name: save deb uses: actions/upload-artifact@v4 diff --git a/policy/templates/releng/.github/workflows/release.yml.d/smoke-tests.gotmpl b/policy/templates/releng/.github/workflows/release.yml.d/smoke-tests.gotmpl index 5986e008..644df7a8 100644 --- a/policy/templates/releng/.github/workflows/release.yml.d/smoke-tests.gotmpl +++ b/policy/templates/releng/.github/workflows/release.yml.d/smoke-tests.gotmpl @@ -21,10 +21,7 @@ echo "::warning file=.github/workflows/release.yml,line=24,col=1,endColumn=8::Using test variation" fi -{{- $pcrepo := $.PCRepo }} -{{if eq .Name "tyk-sink" }} - {{ $pcrepo = print $.PCRepo "-stable" }} -{{ end }} +{{- $uprepo := .Branchvals.Builds.std.UpgradeRepo }} {{ $repo := .Name }} {{ $pcname := .PackageName }} @@ -66,15 +63,15 @@ ARG TARGETARCH COPY {{ $pcname }}*_${TARGETARCH}.deb /{{ $pcname }}.deb RUN apt-get update && apt-get install -y curl - RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/{{ $pcrepo }}/script.deb.sh | bash && apt-get install -y {{ $pcname }}={{ $fromVer }} + RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/{{ $uprepo }}/script.deb.sh | bash && apt-get install -y {{ $pcname }}={{ $fromVer }} RUN dpkg -i /{{ $pcname }}.deb {{ else if eq $distro "rpm" }} ARG RHELARCH COPY {{ $pcname }}*.${RHELARCH}.rpm /{{ $pcname }}.rpm RUN command -v curl || yum install -y curl RUN command -v useradd || yum install -y shadow-utils - RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/{{ $pcrepo }}/script.rpm.sh | bash && yum install -y {{ $pcname }}-{{ $fromVer }}-1 - RUN curl https://keyserver.tyk.io/tyk.io.rpm.signing.key.2020 -o {{ $pcrepo }}.key && rpm --import {{ $pcrepo }}.key + RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/{{ $uprepo }}/script.rpm.sh | bash && yum install -y {{ $pcname }}-{{ $fromVer }}-1 + RUN curl https://keyserver.tyk.io/tyk.io.rpm.signing.key.2020 -o {{ $uprepo }}.key && rpm --import {{ $uprepo }}.key RUN rpm --checksig /{{ $pcname }}.rpm RUN rpm -Uvh --force /{{ $pcname }}.rpm {{ end }} {{/* if $distro */}} @@ -102,9 +99,11 @@ - name: Test the built container image with api functionality test. run: | - docker run -d -p8080:8080 --network {{`${{ job.container.network }}`}} --rm test-{{`${{ matrix.distro }}-${{ matrix.arch }}`}} + docker run -d -p8080:8080 --name=test --platform linux/{{`${{ matrix.arch }}`}} --network {{`${{ job.container.network }}`}} --rm test-{{`${{ matrix.distro }}-${{ matrix.arch }}`}} sleep 2 ./ci/tests/api-functionality/api_test.sh + sleep 2 + docker stop test || true {{- end }} {{- end }} {{/* range distro */}} diff --git a/policy/templates/releng/ci/Dockerfile.std b/policy/templates/releng/ci/Dockerfile.std index 1181030f..9d73c20f 100644 --- a/policy/templates/releng/ci/Dockerfile.std +++ b/policy/templates/releng/ci/Dockerfile.std @@ -2,6 +2,7 @@ FROM {{ .Branchvals.BaseImage }} ARG TARGETARCH +ARG BUILD_PACKAGE_NAME ENV DEBIAN_FRONTEND=noninteractive @@ -25,8 +26,8 @@ RUN rm -rf /root/.cache \ && find /usr/lib -type f -name '*.a' -o -name '*.o' -delete # Comment this to test in dev -COPY *${TARGETARCH}.deb / -RUN rm -f /*fips*.deb && dpkg -i /{{ .PackageName }}*${TARGETARCH}.deb && rm /*.deb +COPY ${BUILD_PACKAGE_NAME}*${TARGETARCH}.deb / +RUN dpkg -i /${BUILD_PACKAGE_NAME}*${TARGETARCH}.deb && rm /*.deb ARG PORTS diff --git a/policy/templates/releng/ci/goreleaser/goreleaser.yml b/policy/templates/releng/ci/goreleaser/goreleaser.yml index 22e144d1..fca88197 100644 --- a/policy/templates/releng/ci/goreleaser/goreleaser.yml +++ b/policy/templates/releng/ci/goreleaser/goreleaser.yml @@ -16,15 +16,7 @@ before: - ./ci/copy-framework-files.sh {{- end}} -{{- if .Branchvals.Cgo }} -{{- template "cgo_builds" . }} -{{- else }} {{- template "builds" . }} -{{- end }} - -{{- template "nfpm" . }} - -{{- template "publishers" . }} # This disables archives archives: diff --git a/policy/templates/subtemplates/auto/auto-test.gotmpl b/policy/templates/subtemplates/auto/auto-test.gotmpl index af698875..039a14fb 100644 --- a/policy/templates/subtemplates/auto/auto-test.gotmpl +++ b/policy/templates/subtemplates/auto/auto-test.gotmpl @@ -100,7 +100,7 @@ {{- if eq .dot.Name "tyk-pro" }} tags=($match_tag) {{ else }} - tags=({{`${{ needs.goreleaser.outputs.tags }}`}}) + tags=({{`${{ needs.goreleaser.outputs.std_tags }}`}}) {{ end -}} set -eaxo pipefail docker run -q --rm -v ~/.docker/config.json:/root/.docker/config.json tykio/gromit policy match ${tags[0]} ${match_tag} 2>versions.env diff --git a/policy/templates/subtemplates/goreleaser.yml.d/builds.gotmpl b/policy/templates/subtemplates/goreleaser.yml.d/builds.gotmpl index c3bd1b0d..ebb7b06e 100644 --- a/policy/templates/subtemplates/goreleaser.yml.d/builds.gotmpl +++ b/policy/templates/subtemplates/goreleaser.yml.d/builds.gotmpl @@ -1,29 +1,164 @@ {{define "builds"}} +{{ $r := . }} + builds: - - id: std +{{- range $b, $bv := .Branchvals.Builds }} +{{- range $a := $bv.Archs }} + - id: {{ printf "%s-%s" $b $a.Go }} + {{- if $bv.Flags }} + flags: + {{- range $flag := $bv.Flags }} + - {{ $flag }} + {{- end }} {{/* range */}} + {{- end }} {{/* if */}} + env: + - NOP=nop # ignore this, it is jsut to avoid a complex conditional in the templates + {{- if $r.Branchvals.Cgo }} + - CC={{ $r.GetCC $a.Deb "amd64" }} + {{- end }} + {{- range $env := $bv.Env }} + - $env + {{- end }} ldflags: - - -X {{.Branchvals.VersionPackage}}.Version={{`{{.Version}}`}} - - -X {{.Branchvals.VersionPackage }}.Commit={{`{{.FullCommit}}`}} - - -X {{.Branchvals.VersionPackage}}.BuildDate={{`{{.Date}}`}} - - -X {{.Branchvals.VersionPackage}}.BuiltBy=goreleaser + - -X {{$r.Branchvals.VersionPackage}}.Version={{`{{.Version}}`}} + - -X {{$r.Branchvals.VersionPackage}}.Commit={{`{{.FullCommit}}`}} + - -X {{$r.Branchvals.VersionPackage}}.BuildDate={{`{{.Date}}`}} + - -X {{$r.Branchvals.VersionPackage}}.BuiltBy=goreleaser goos: - linux goarch: - - amd64 - - arm64 -{{ if has "s390x" .Branchvals.Features }} - - s390x -{{ end }} - binary: {{.Binary}} - - id: fips - ldflags: - - -X {{.Branchvals.VersionPackage}}.Version={{`{{.Version}}`}} - - -X {{.Branchvals.VersionPackage }}.Commit={{`{{.FullCommit}}`}} - - -X {{.Branchvals.VersionPackage}}.BuildDate={{`{{.Date}}`}} - - -X {{.Branchvals.VersionPackage}}.BuiltBy=goreleaser - goos: - - linux - goarch: - - amd64 - binary: {{.Binary}} + - {{ $a.Go }} + binary: {{$r.Binary}} + {{ end }} {{/* range $bv.Archs */}} +{{ end }} {{/* range builds .Branchvals.Builds */}} + +nfpms: + {{- range $b, $bv := .Branchvals.Builds }} + - id: {{ $b }} + vendor: "Tyk Technologies Ltd" + homepage: "https://tyk.io" + maintainer: "Tyk " + description: {{ $bv.Description }} + package_name: {{ $bv.BuildPackageName }} + file_name_template: "{{`{{ .ConventionalFileName }}`}}" + builds: + {{- range $a := $bv.Archs }} + - {{ printf "%s-%s" $b $a.Go }} + {{- end }} + formats: + - deb + - rpm + contents: + - src: "README.md" + dst: "/opt/share/docs/{{ $r.PackageName }}/README.md" + - src: "ci/install/*" + dst: "/opt/{{ $r.PackageName }}/install" + - src: ci/install/inits/systemd/system/{{ $r.PackageName }}.service + dst: /lib/systemd/system/{{ $r.PackageName }}.service + - src: ci/install/inits/sysv/init.d/{{ $r.PackageName }} + dst: /etc/init.d/{{ $r.PackageName }} + {{- if eq $r.Name "tyk-analytics" }} + - src: /opt/{{ $r.PackageName }} + dst: /opt/{{ $r.Name }} + type: "symlink" + - src: "EULA.md" + dst: "/opt/share/docs/{{ $r.PackageName }}/EULA.md" + - src: "portal/*" + dst: "/opt/{{ $r.PackageName }}/portal" + - src: "utils/scripts/*" + dst: "/opt/{{ $r.PackageName }}/utils/scripts" + - src: "schemas/*" + dst: "/opt/{{ $r.PackageName }}/schemas" + - src: "webclient/lang/*" + dst: "/opt/{{ $r.PackageName }}/lang" + - src: tyk_config_sample.config + dst: /opt/{{ $r.PackageName }}/{{ $r.Branchvals.ConfigFile }} + type: "config|noreplace" + {{- end}} + {{- if eq $r.Name "tyk" }} + - src: /opt/{{ $r.PackageName }} + dst: /opt/{{ $r.Name }} + type: "symlink" + - src: "LICENSE.md" + dst: "/opt/share/docs/{{ $r.PackageName }}/LICENSE.md" + - src: "apps/app_sample.*" + dst: "/opt/{{ $r.PackageName }}/apps" + - src: "templates/*.json" + dst: "/opt/{{ $r.PackageName }}/templates" + - src: "templates/playground/*" + dst: "/opt/{{ $r.PackageName }}/templates/playground" + - src: "middleware/*.js" + dst: "/opt/{{ $r.PackageName }}/middleware" + - src: "event_handlers/sample/*.js" + dst: "/opt/{{ $r.PackageName }}/event_handlers/sample" + - src: "policies/*.json" + dst: "/opt/{{ $r.PackageName }}/policies" + - src: "coprocess/*" + dst: "/opt/{{ $r.PackageName }}/coprocess" + - src: tyk.conf.example + dst: /opt/{{ $r.PackageName }}/{{ $r.Branchvals.ConfigFile }} + type: "config|noreplace" + {{- end }} + {{- if eq $r.Name "tyk-identity-broker" }} + - src: "LICENSE.md" + dst: "/opt/share/docs/{{ $r.PackageName }}/LICENSE.md" + - src: tib_sample.conf + dst: /opt/{{ $r.PackageName }}/{{ $r.Branchvals.ConfigFile }} + type: "config|noreplace" + {{- end }} + {{- if eq $r.Name "tyk-pump" }} + - src: "LICENSE.md" + dst: "/opt/share/docs/{{ $r.PackageName }}/LICENSE.md" + - src: pump.example.conf + dst: /opt/{{ $r.PackageName }}/{{ $r.Branchvals.ConfigFile }} + type: "config|noreplace" + {{- end }} + {{- if eq $r.Name "tyk-sink" }} + - src: tyk_sink_sample.conf + dst: /opt/{{ $r.PackageName }}/{{ $r.Branchvals.ConfigFile }} + type: "config|noreplace" + {{- end }} + {{- if eq $r.Name "portal" }} + - src: portal.conf + dst: /opt/{{ $r.PackageName }}/{{ $r.Branchvals.ConfigFile }} + type: "config|noreplace" + - src: app/views + dst: /opt/{{ $r.PackageName }}/app/views + - src: bootstrap/views + dst: /opt/{{ $r.PackageName }}/bootstrap/views + - src: default.zip + dst: /opt/{{ $r.PackageName }}/default.zip + - src: public/system + dst: /opt/{{ $r.PackageName }}/public/system + - src: ci/entrypoint.sh + dst: /opt/{{ $r.PackageName }}/entrypoint.sh + file_info: + mode: 0775 + {{- end }} + scripts: + preinstall: "ci/install/before_install.sh" + postinstall: "ci/install/post_install.sh" + postremove: "ci/install/post_remove.sh" + bindir: "/opt/{{ $r.PackageName }}" + rpm: + scripts: + posttrans: ci/install/post_trans.sh + signature: + key_file: tyk.io.signing.key + deb: + signature: + key_file: tyk.io.signing.key + type: origin +{{- end }} {{/* range nfpm Branchvals.Builds */}} + +publishers: +{{- range $b, $bv := .Branchvals.Builds }} + - name: {{ $b }} + ids: + - {{ $b }} + env: + - PACKAGECLOUD_TOKEN={{`{{ .Env.PACKAGECLOUD_TOKEN }}`}} + cmd: {{`packagecloud publish --debvers "{{ .Env.DEBVERS }}" --rpmvers "{{ .Env.RPMVERS }}"`}} tyk/{{ $bv.PCRepo }} {{`{{ .ArtifactPath }}`}} +{{- end }} {{/* range publishers */}} + {{end}} diff --git a/policy/templates/subtemplates/goreleaser.yml.d/cgo_builds.gotmpl b/policy/templates/subtemplates/goreleaser.yml.d/cgo_builds.gotmpl deleted file mode 100644 index a2fe6056..00000000 --- a/policy/templates/subtemplates/goreleaser.yml.d/cgo_builds.gotmpl +++ /dev/null @@ -1,79 +0,0 @@ -{{define "cgo_builds"}} -{{ $r := . }} -{{ $has_ee := has "ee" $r.Branchvals.Features }} -{{ $has_fips := has "fips" $r.Branchvals.Features }} - -{{ $builds := list "std" }} - -{{ if $has_fips }} - {{ $builds = append $builds "fips" }} -{{ end }} - -{{ if $has_ee }} - {{ $builds = append $builds "ee" }} -{{ end }} - -builds: - {{- range $b := $builds }} - - id: {{ $b }} - flags: - - -tags=ignore - - -trimpath - - -tags=goplugin{{ if eq $b "fips" }},fips,boringcrypto{{ if $has_ee }},ee{{ end }}{{ else if eq $b "ee" }},ee{{ end }} - {{ if eq $b "fips" }} - env: - - GOEXPERIMENT=boringcrypto - {{ end }} - ldflags: - - -X {{$r.Branchvals.VersionPackage}}.Version={{`{{.Version}}`}} - - -X {{$r.Branchvals.VersionPackage}}.Commit={{`{{.FullCommit}}`}} - - -X {{$r.Branchvals.VersionPackage}}.BuildDate={{`{{.Date}}`}} - - -X {{$r.Branchvals.VersionPackage}}.BuiltBy=goreleaser - goos: - - linux - goarch: - - amd64 - binary: {{$r.Binary}} - - {{ if ne $b "fips" }} - - id: {{ $b }}-arm64 - flags: - - -tags=ignore - - -trimpath - - -tags=goplugin{{ if eq $b "ee" }},ee{{ end }} - ldflags: - - -X {{$r.Branchvals.VersionPackage}}.Version={{`{{.Version}}`}} - - -X {{$r.Branchvals.VersionPackage}}.Commit={{`{{.FullCommit}}`}} - - -X {{$r.Branchvals.VersionPackage}}.BuildDate={{`{{.Date}}`}} - - -X {{$r.Branchvals.VersionPackage}}.BuiltBy=goreleaser - env: - - CC=aarch64-linux-gnu-gcc - goos: - - linux - goarch: - - arm64 - binary: {{$r.Binary}} - - {{ if has "s390x" $r.Branchvals.Features -}} - - id: {{ $b }}-s390x - flags: - - -tags=ignore - - -trimpath - - -tags=goplugin{{ if eq $b "ee" }},ee{{ end }} - ldflags: - - -X {{$r.Branchvals.VersionPackage}}.Version={{`{{.Version}}`}} - - -X {{$r.Branchvals.VersionPackage}}.Commit={{`{{.FullCommit}}`}} - - -X {{$r.Branchvals.VersionPackage}}.BuildDate={{`{{.Date}}`}} - - -X {{$r.Branchvals.VersionPackage}}.BuiltBy=goreleaser - env: - - CC=s390x-linux-gnu-gcc - goos: - - linux - goarch: - - s390x - binary: {{$r.Binary}} - {{ end }} {{/* s390x */}} - {{ end }} {{/* !fips */}} - - {{ end }} {{/* range over $builds */}} -{{end}} diff --git a/policy/templates/subtemplates/goreleaser.yml.d/nfpm.gotmpl b/policy/templates/subtemplates/goreleaser.yml.d/nfpm.gotmpl index c60a571c..10f044ff 100644 --- a/policy/templates/subtemplates/goreleaser.yml.d/nfpm.gotmpl +++ b/policy/templates/subtemplates/goreleaser.yml.d/nfpm.gotmpl @@ -1,36 +1,17 @@ {{define "nfpm" }} {{ $r := . }} -{{ $has_ee := has "ee" $r.Branchvals.Features }} -{{ $has_fips := has "fips" $r.Branchvals.Features }} - -{{ $builds := list "std" }} - -{{ if $has_fips }} - {{ $builds = append $builds "fips" }} -{{ end }} - -{{ if $has_ee }} - {{ $builds = append $builds "ee" }} -{{ end }} - nfpms: - {{ range $b := $builds }} + {{- range $b, $bv := .Branchvals.Builds }} - id: {{ $b }} vendor: "Tyk Technologies Ltd" homepage: "https://tyk.io" maintainer: "Tyk " - description: {{ $r.Description }} - package_name: {{ if eq $b "fips" }}{{ $r.PackageName }}-fips{{ else }}{{ $r.PackageName }}{{ end }}{{ if eq $b "ee" }}-ee{{ end }} + description: {{ $bv.Description }} + package_name: {{ $bv.BuildPackageName }} file_name_template: "{{`{{ .ConventionalFileName }}`}}" builds: - {{- if and $r.Branchvals.Cgo (ne $b "fips")}} - - {{ $b }} - - {{ $b }}-arm64 - {{ if has "s390x" $r.Branchvals.Features }} - - {{ $b }}-s390x - {{- end }} - {{- else }} - - {{ $b }} + {{- range $a := $bv.Archs }} + - {{ printf "%s-%s" $b $a.Go }} {{- end }} formats: - deb @@ -136,5 +117,5 @@ nfpms: signature: key_file: tyk.io.signing.key type: origin - {{- end }} {{/* range over $builds */}} + {{- end }} {{/* range over Branchvals.Builds */}} {{- end }} diff --git a/policy/templates/subtemplates/goreleaser.yml.d/publishers.gotmpl b/policy/templates/subtemplates/goreleaser.yml.d/publishers.gotmpl deleted file mode 100644 index b1dbdd6b..00000000 --- a/policy/templates/subtemplates/goreleaser.yml.d/publishers.gotmpl +++ /dev/null @@ -1,7 +0,0 @@ -{{define "publishers"}} -publishers: - - name: {{ .PCRepo }}-unstable - env: - - PACKAGECLOUD_TOKEN={{`{{ .Env.PACKAGECLOUD_TOKEN }}`}} - cmd: {{`packagecloud publish --debvers "{{ .Env.DEBVERS }}" --rpmvers "{{ .Env.RPMVERS }}"`}} tyk/{{ .PCRepo }}-unstable {{`{{ .ArtifactPath }}`}} -{{end}} diff --git a/policy/templates/test-square/.github/workflows/test-square.yml b/policy/templates/test-square/.github/workflows/test-square.yml index 4f33ed7e..61d73cc5 100644 --- a/policy/templates/test-square/.github/workflows/test-square.yml +++ b/policy/templates/test-square/.github/workflows/test-square.yml @@ -23,7 +23,7 @@ on: {{- end }} env: - VARIATION: prod-variations + VARIATION: prod-variation BASE_REF: {{`${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name}}`}} jobs: diff --git a/policy/tfuncs.go b/policy/tfuncs.go new file mode 100644 index 00000000..164a5497 --- /dev/null +++ b/policy/tfuncs.go @@ -0,0 +1,62 @@ +package policy + +import ( + "reflect" + + "github.com/TykTechnologies/gromit/util" +) + +// Template functions called while rendering + +// getCC returns the appropriate C compiler for the target architecture given the host architecture +func (rp RepoPolicy) GetCC(target, host string) string { + if target != host { + return target + "-linux-gnu-gcc" + } + return "gcc" +} + +// getImages returns the list of container manifests +func (rp RepoPolicy) GetImages(repos ...string) []string { + images := make(util.Set[string]) + for _, bv := range rp.Branchvals.Builds { + for _, repo := range repos { + image := getBuildField(bv, repo) + if len(image) > 0 { + images.Add(image) + } + } + } + return images.Members() +} + +// getDockerPlatforms returns the list of docker platforms that are to be supported +func (rp RepoPolicy) GetDockerPlatforms() []string { + platforms := make(util.Set[string]) + for _, bv := range rp.Branchvals.Builds { + for _, a := range bv.Archs { + if len(a.Docker) > 0 { + platforms.Add(a.Docker) + } + } + } + return platforms.Members() +} + +// getDockerBuilds returns a map of builds that have at least one container build +func (rp RepoPolicy) GetDockerBuilds() buildMap { + dBuilds := make(buildMap) + for b, bv := range rp.Branchvals.Builds { + if bv.CIRepo != "" || bv.DHRepo != "" || bv.CSRepo != "" { + dBuilds[b] = bv + } + } + return dBuilds +} + +// getBuildField helps with accessing properties of the build type +func getBuildField(v *build, field string) string { + r := reflect.ValueOf(v) + f := reflect.Indirect(r).FieldByName(field) + return f.String() +} diff --git a/testdata/config-test.yaml b/testdata/config-test.yaml index cabc1e96..a018a8d7 100644 --- a/testdata/config-test.yaml +++ b/testdata/config-test.yaml @@ -7,9 +7,8 @@ policy: - a builenv: wrong repos: + # Use this repo to test features repo0: - description: >- - Repo Zero buildenv: stillwrong configfile: repo0.conf features: @@ -25,20 +24,46 @@ policy: features: - e - f + # Use this to test builds repo1: - description: >- - Repo One buildenv: wrong + builds: + std: + flags: + - flagstd1 + buildpackagename: repo1-pkg + dhrepo: repo1-doc-wrong + archs: + - go: go1 + deb: deb1 + docker: doc1 + std2: + flags: + - flag2 branches: main: buildenv: right features: - e + builds: + std: + flags: + - flagstd2 + dhrepo: repo1-doc-right + archs: + - go: go2 + deb: deb2 + docker: doc2 + std2: + buildpackagename: repo1-std2 grp1: features: - a builenv: maybe repos: repo3: - description: >- - Repo Three + branches: + master: + builds: + std3: + buildpackagename: repo3-std3