Introduce the build concept

A new key Builds at the repo and branch levels contains the builds for
a repo.
Upgrade tests use the std build artefacts.

Author: Alok G Singh

config/config.yaml

@@ -1,6 +1,6 @@
 module github.com/TykTechnologies/gromit
 
-go 1.22.0
+go 1.23
 
 require (
 	github.com/Masterminds/sprig/v3 v3.3.0

go.mod

@@ -55,6 +55,7 @@ func TestBundleRender(t *testing.T) {
 			if err != nil {
 				t.Fatalf("Error creating temp dir: %v", err)
 			}
+			t.Logf("templates rendered to: %s", tmpDir)
 			defer os.RemoveAll(tmpDir)
 
 			err = rp.SetBranch("master")

policy/bundle_test.go

@@ -1,16 +1,18 @@
 package policy
 
 import (
-	"bytes"
 	"fmt"
 	"os"
 	"path/filepath"
+	"slices"
 	"time"
 
+	"maps"
+
+	"dario.cat/mergo"
 	"github.com/jinzhu/copier"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
-	"golang.org/x/exp/maps"
 )
 
 // repoConfig contains all the attributes of a repo. Each element here
@@ -20,15 +22,11 @@ import (
 // levels
 type repoConfig struct {
 	Owner               string
-	Description         string
-	PCRepo              string
-	DHRepo              string
-	CSRepo              string
-	PackageName         string
-	Reviewers           []string
 	ExposePorts         string
+	PackageName         string
 	Binary              string
 	Buildenv            string
+	Builds              buildMap
 	BaseImage           string
 	DistrolessBaseImage string
 	Cgo                 bool
@@ -42,6 +40,26 @@ type repoConfig struct {
 	Repos               map[string]repoConfig `copier:"-"`
 }
 
+// build models the variations in build and their corresponding packages
+type build struct {
+	Flags            []string
+	BuildPackageName string
+	Description      string
+	ImageTitle       string
+	PCRepo           string
+	DHRepo           string
+	CSRepo           string
+	CIRepo           string
+	Env              []string
+	Archs            []struct {
+		Docker string
+		Deb    string
+		Go     string
+	}
+}
+
+type buildMap map[string]*build
+
 // Policies models the config file structure. There are three levels
 // at which a particular value can be set: group-level, repo, branch.
 // The group level is applicable for all the repos in that group.
@@ -66,6 +84,7 @@ type branchVals struct {
 	UpgradeFromVer      string
 	Tests               []string
 	Features            []string
+	Builds              buildMap
 	DeletedFiles        []string
 }
 
@@ -77,13 +96,10 @@ type branchVals struct {
 type RepoPolicy struct {
 	Owner          string
 	Name           string
-	Description    string
 	Default        string
-	PCRepo         string
-	DHRepo         string
-	CSRepo         string
-	Binary         string
 	PackageName    string
+	Binary         string
+	Builds         buildMap
 	Reviewers      []string
 	ExposePorts    string
 	Cgo            bool
@@ -93,9 +109,7 @@ type RepoPolicy struct {
 	Branch         string
 	Branchvals     branchVals
 	Branches       map[string]branchVals
-	prBranch       string
 	Timestamp      string
-	Visibility     string
 }
 
 // PushOptions collects the input required to update templates for a
@@ -115,7 +129,6 @@ func (rp *RepoPolicy) SetTimestamp(ts time.Time) {
 		ts = time.Now().UTC()
 	}
 	rp.Timestamp = ts.Format(time.UnixDate)
-
 }
 
 // GetTimeStamp returns the timestamp currently set for the given repopolicy.
@@ -126,7 +139,7 @@ func (rp *RepoPolicy) GetTimeStamp() (time.Time, error) {
 	return ts, err
 }
 
-// SetBranch sets the Branch and Branchvals properties so that templates can simply access them instead of looking them up in the Branches map
+// SetBranch sets the Branch and Branchvals properties so that templates can simply access them instead of looking them up in the Branches map. This must be called before calling Render()
 func (rp *RepoPolicy) SetBranch(branch string) error {
 	bv, found := rp.Branches[branch]
 	if !found {
@@ -140,7 +153,7 @@ func (rp *RepoPolicy) SetBranch(branch string) error {
 
 // GetAllBranches returns all the branches that are managed for this repo
 func (rp *RepoPolicy) GetAllBranches() []string {
-	return maps.Keys(rp.Branches)
+	return slices.Sorted(maps.Keys(rp.Branches))
 }
 
 // GetRepoPolicy will fetch the RepoPolicy for the supplied repo with
@@ -193,6 +206,9 @@ func (p *Policies) GetRepoPolicy(repo string) (RepoPolicy, error) {
 		if err != nil {
 			return rp, err
 		}
+		// builds are merged
+		log.Debug().Msgf("Merging builds for %s/%s", rp.Name, b)
+		rbv.Builds = mergeBuilds(r.Builds, bbv.Builds)
 		// attributes that are unions
 		rbv.Features = newSetFromSlices(group.Features, r.Features, bbv.Features).Members()
 		rbv.DeletedFiles = newSetFromSlices(p.DeletedFiles, group.DeletedFiles, r.DeletedFiles, bbv.DeletedFiles).Members()
@@ -204,6 +220,16 @@ func (p *Policies) GetRepoPolicy(repo string) (RepoPolicy, error) {
 	return rp, nil
 }
 
+// mergeBuilds returns a merged build map from _r_epo and _b_ranch level
+func mergeBuilds(r, b buildMap) buildMap {
+	merged := make(buildMap)
+	maps.Copy(merged, r)
+	if err := mergo.Merge(&merged, b, mergo.WithOverride, mergo.WithAppendSlice); err != nil {
+		log.Fatal().Interface("dst", merged).Interface("src", b).Msgf("could not merge branch-level build definitions for: %v", err)
+	}
+	return merged
+}
+
 // ProcessBranch will render the templates into a git worktree for the supplied branch, commit and push the changes upstream
 // The upstream branch name is the supplied branch name prefixed with releng/ and is returned
 func (rp *RepoPolicy) ProcessBranch(pushOpts *PushOptions) error {
@@ -270,36 +296,6 @@ func (rp *RepoPolicy) ProcessBranch(pushOpts *PushOptions) error {
 	return nil
 }
 
-// Stringer implementation for Policies
-func (p Policies) String() string {
-	w := new(bytes.Buffer)
-	for _, grp := range p.Groups {
-		for repo, crPol := range grp.Repos {
-			fmt.Fprintf(w, "%s: package %s, image %s", repo, crPol.PackageName, crPol.DHRepo)
-			rp, err := p.GetRepoPolicy(repo)
-			if err != nil {
-				log.Fatal().Str("repo", repo).Err(err).Msg("failed to get policy, this should not happen")
-			}
-			fmt.Fprintf(w, " %s\n", rp)
-		}
-	}
-	return w.String()
-}
-
-// Stringer implementation for RepoPolicy
-func (rp RepoPolicy) String() string {
-	w := new(bytes.Buffer)
-	for b, bv := range rp.Branches {
-		fmt.Fprintf(w, " %s: package %s, image %s, features %v", b, rp.PackageName, rp.DHRepo, bv.Features)
-		if len(bv.Buildenv) > 0 {
-			fmt.Fprintf(w, " built on %s", bv.Buildenv)
-		} else {
-			fmt.Fprintf(w, " not built")
-		}
-	}
-	return w.String()
-}
-
 // LoadRepoPolicies populates the supplied policies with the policy key from a the config file
 // This will panic if the type assertions fail
 func LoadRepoPolicies(policies *Policies) error {

policy/policy.go

@@ -29,21 +29,38 @@ func TestPolicyConfig(t *testing.T) {
 	assert.EqualValues(t, "right", repo0.Branchvals.Buildenv, "testing branch-level override for main")
 	assert.ElementsMatch(t, []string{"a", "b", "c", "d"}, repo0.Branchvals.Features, "testing merging of branchvals")
 	assert.EqualValues(t, "repo0.conf", repo0.Branchvals.ConfigFile, "testing repo-level inheritance of branchvals")
-	assert.EqualValues(t, "Repo Zero", repo0.Description, "testing repo-level value")
 
 	err = repo0.SetBranch("dev")
 	if err != nil {
 		t.Fatalf("Could not set dev branch: %v", err)
 	}
 	assert.EqualValues(t, "stillright", repo0.Branchvals.Buildenv, "testing overrides for dev")
 	assert.ElementsMatch(t, []string{"a", "b", "e", "f"}, repo0.Branchvals.Features, "testing merging")
-	assert.EqualValues(t, "Repo Zero", repo0.Description, "testing inheritance")
 
 	repo1, err := pol.GetRepoPolicy("repo1")
 	if err != nil {
 		t.Fatalf("Could not get repo1: %v", err)
 	}
-	assert.EqualValues(t, "Repo One", repo1.Description, "testing second repo")
 	r1b := repo1.GetAllBranches()
 	assert.EqualValues(t, []string{"main"}, r1b, "testing branches")
+	err = repo1.SetBranch("main")
+	if err != nil {
+		t.Fatalf("Could not set main branch: %v", err)
+	}
+	assert.EqualValues(t, []string{"flagstd1", "flagstd2"}, repo1.Branchvals.Builds["std"].Flags, "testing explicit merge")
+	assert.EqualValues(t, "repo1-std2", repo1.Branchvals.Builds["std2"].BuildPackageName, "testing implicit merges at branch")
+	assert.EqualValues(t, []string{"flag2"}, repo1.Branchvals.Builds["std2"].Flags, "testing implicit merge from repo")
+	assert.EqualValues(t, build{Flags: []string{"flagstd1", "flagstd2"},
+		BuildPackageName: "repo1-pkg",
+		DHRepo:           "repo1-doc-right",
+		Archs: []struct {
+			Docker string
+			Deb    string
+			Go     string
+		}{
+			{"doc1", "deb1", "go1"},
+			{"doc2", "deb2", "go2"}},
+	}, *repo1.Branchvals.Builds["std"], "testing full merge")
+	assert.EqualValues(t, []string{"repo1-doc-right"}, repo1.GetImages("DHRepo"), "testing getImages()")
+	assert.EqualValues(t, []string{"doc1", "doc2"}, repo1.GetDockerPlatforms(), "testing getDockerPlatforms()")
 }

policy/policy_test.go

@@ -6,8 +6,8 @@ ARG EDITION
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-COPY *${TARGETARCH}.deb /
-RUN rm -f /*fips*.deb && dpkg -i /{{ .PackageName }}${EDITION}_*${TARGETARCH}.deb && rm /*.deb
+COPY ${BUILD_PACKAGE_NAME}*${TARGETARCH}.deb /
+RUN dpkg -i /${BUILD_PACKAGE_NAME}_*${TARGETARCH}.deb && rm /*.deb
 
 FROM gcr.io/distroless/{{ .Branchvals.DistrolessBaseImage }}
 

policy/templates/distroless/ci/Dockerfile.distroless

@@ -4,18 +4,7 @@
 # - arm64
 # - amd64
 
-{{- if has "el7-pgo-build" .Branchvals.Features }}
-  {{- template "cgo_builds" . }}
-{{- else }}
-  {{- template "builds" . }}
-{{- end }}
-{{- template "nfpm" . }}
-
-publishers:
-  - name: {{ .PCRepo }}-unstable
-    env:
-      - PACKAGECLOUD_TOKEN={{`{{ .Env.PACKAGECLOUD_TOKEN }}`}}
-    cmd:  {{`packagecloud publish --debvers "{{ .Env.DEBVERS }}" --rpmvers "{{ .Env.RPMVERS }}"`}} tyk/{{ .PCRepo }}-unstable {{`{{ .ArtifactPath }}`}}
+{{- template "builds" . }}
 
 # This disables archives
 archives: