diff --git a/Dockerfile b/Dockerfile index 98c41bf..8466dd1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,8 @@ COPY cmd/ cmd/ COPY util/ util/ COPY web/ web/ COPY swarmcd/ swarmcd/ -RUN CGO_ENABLED=0 GOOS=linux go build -o /swarm-cd ./cmd/ +RUN CGO_ENABLED=0 GOOS=linux go build -o /swarm-cd ./cmd/swarm-cd +RUN CGO_ENABLED=0 GOOS=linux go build -o /template-gen ./cmd/template-gen RUN go test ./swarmcd/ # Stage 3: Final production image (depends on previous stages) @@ -26,6 +27,7 @@ WORKDIR /app RUN apk add --no-cache ca-certificates && update-ca-certificates # Copy the built backend binary from the backend build stage COPY --from=backend-build /swarm-cd /app/ +COPY --from=backend-build /template-gen /app/ # Copy the built frontend from the frontend build stage COPY --from=frontend-build /ui/dist/ /app/ui/ # Sets the web server mode to release diff --git a/README.md b/README.md index f698638..ad64cb5 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,101 @@ Please note that: - if the global setting is set to `true`, it ignores individual stacks overrides. - if the stack-level setting is set to `true`, it ignores the `sops_files` setting altogether. +## Configuring stacks + +### Using variables from a value file +When defining a stack, you can provide a path to a yaml file with variables by setting the `value_file` field. +Variables defined in this file can then be accessed in the compose file. + +[Sprig](https://masterminds.github.io/sprig/) functions can be used in the compose files. + +```yaml +# value_file.yaml +--- +myvar: myvalue +mylist: + - a + - b + - c +``` + +```yaml +# compose.yaml +services: + foo: + image: foo + environment: + MY_VAR: "{{ .Values.myvar }}" + MY_LIST: "{{ .Values.mylist | join ":" }}" # == "a:b:c" +``` + +### Defining global variables + +Variables can be defined for all stack in a `global_values.yaml` file, or directly in the `config.yaml` file using the `global_values` field. + +They can be overriden using a stack value file. + + +```yaml +# global_values.yaml +--- +var_a: fromglobal +var_b: fromglobal +``` + +```yaml +# value_file.yaml +--- +var_b: overriden +``` + +```yaml +# compose.yaml +services: + foo: + image: foo + environment: + MY_VAR_A: "{{ .Values.var_a }}" # == fromglobal + MY_VAR_B: "{{ .Values.var_b }}" # == overriden +``` + +### Templating + +Templates are automatically imported from the `template` folder. Swarmcd will read any file with the `.tmpl` extension. For a syntax breakdown, see the [official go documentation](https://pkg.go.dev/text/template). +They can be used in compose files like so: + +```yaml +# template/storage.tmpl + +{{- define "nfs_volume" }} + {{ .name }}-vol: + driver_opts: + type: nfs + o: addr=1.2.3.4,nfsvers=4 + device: :/path/to/{{ .name }} +{{- end }} +``` + +```yaml +# compose.yaml +services: + foo: + image: foo + volumes: + - foo-vol:/etc/foo + +volumes: +{{ template "nfs_volume" (dict "name" "foo") }} +``` + +### Testing template generation + +For debugging purpose, you can generate your compose without a running instance of swarm-cd. For that, you can call `/app/template-gen, available in the docker image. For example: +``` sh +docker run -v $(pwd)/testdata:/data --rm -it ghcr.io/m-adawi/swarm-cd:latest /app/template-gen --valuefile /data/values.yml /data/compose.yml out.yaml +``` +For a list of all available flags, use the `--help` flag. + ## Connect SwarmCD to a remote docker socket You can use the `DOCKER_HOST` environment variable to point SwarmCD to a remote docker socket, diff --git a/cmd/main.go b/cmd/main.go deleted file mode 100644 index 37aeb6f..0000000 --- a/cmd/main.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -import ( - "fmt" - "os" - - "github.com/m-adawi/swarm-cd/swarmcd" - "github.com/m-adawi/swarm-cd/util" - "github.com/m-adawi/swarm-cd/web" -) - -func init() { - err := util.LoadConfigs() - handleInitError(err) - err = swarmcd.Init() - handleInitError(err) -} - -func main() { - go swarmcd.Run() - if err := web.RunServer(util.Configs.Address); err != nil { - fmt.Println(err) - os.Exit(1) - } -} - -func handleInitError(err error) { - if err != nil { - fmt.Println(err) - os.Exit(1) - } -} diff --git a/cmd/template-gen/main.go b/cmd/template-gen/main.go new file mode 100644 index 0000000..9644e99 --- /dev/null +++ b/cmd/template-gen/main.go @@ -0,0 +1,81 @@ +package main + +import ( + "log" + "flag" + "fmt" + "os" + + "github.com/m-adawi/swarm-cd/swarmcd" + "github.com/m-adawi/swarm-cd/util" +) + +func init() { + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [OPTIONS]... INPUTFILE [OUTPUTFILE]\n", os.Args[0]) + fmt.Fprintf(flag.CommandLine.Output(), "If OUTPUTFILE is not provided or is equal to \"-\", result will be outputted to stdout\n\n") + flag.PrintDefaults() + } +} + +func main() { + var valueFile, globalPath, configPath, templateFolder string + flag.StringVar(&valueFile, "valuefile", "", "Path to a value file") + flag.StringVar(&globalPath, "global", "", "Path to a global value file") + flag.StringVar(&configPath, "config", "", "Path to a config file (for globals)") + flag.StringVar(&templateFolder, "templatefolder", "", "Path to the template folder") + + flag.Parse() + + if len(flag.Args()) == 0 { + log.Fatal("INPUTFILE is required") + } + composeFile := flag.Args()[0] + var err error + var globalValuesMap map[string]any + if configPath != "" { + err = util.ReadConfig(configPath) + if err != nil { + log.Fatal("Could not parse config file: ", err) + } + globalValuesMap = util.Configs.GlobalValues + } else if globalPath != "" { + globalValuesMap, err = util.ParseValuesFile(globalPath, "global") + if err != nil { + log.Fatal("Could not parse global file: ", err) + } + } + + + outputFile := "-" + if len(flag.Args()) > 1 { + outputFile = flag.Args()[1] + } + + stack := swarmcd.NewSwarmStack( + "Template test", + nil, + "nobranch", + composeFile, + nil, + valueFile, + false, + globalValuesMap, + templateFolder, + ) + + stackBytes, err := stack.GenerateStack() + if err != nil { + log.Fatal("Could not generate stack: ", err) + } + if outputFile == "-" { + fmt.Println(string(stackBytes)) + + } else { + err = os.WriteFile(outputFile, stackBytes, 0666) + if err != nil { + log.Fatal("Could not write file ", outputFile, ": ", err) + } + } +} + diff --git a/docs/config.yaml b/docs/config.yaml index 4c4a4b5..f421c3a 100644 --- a/docs/config.yaml +++ b/docs/config.yaml @@ -25,3 +25,8 @@ stacks: # The WEB UI address address: 0.0.0.0:8080 + +# If you need the same values on every stack (domain names), +# you can define global values here instead of global_values.yaml file +# They can be overriden in the values file at stack level. +global_values: diff --git a/go.mod b/go.mod index 4e387ea..4642eb1 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/m-adawi/swarm-cd go 1.22.5 require ( + github.com/Masterminds/sprig/v3 v3.2.1 github.com/docker/cli v27.0.3+incompatible github.com/getsops/sops/v3 v3.9.0 github.com/gin-gonic/gin v1.10.0 @@ -27,6 +28,8 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.1.1 // indirect github.com/aws/aws-sdk-go-v2 v1.30.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect github.com/aws/aws-sdk-go-v2/config v1.27.21 // indirect @@ -80,6 +83,8 @@ require ( github.com/hashicorp/go-sockaddr v1.0.6 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/vault/api v1.14.0 // indirect + github.com/huandu/xstrings v1.3.2 // indirect + github.com/imdario/mergo v0.3.11 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect @@ -89,9 +94,11 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mitchellh/reflectwalk v1.0.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect @@ -100,6 +107,7 @@ require ( github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/shopspring/decimal v1.2.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect diff --git a/go.sum b/go.sum index a9005fe..61debfa 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,12 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mx github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/Masterminds/sprig/v3 v3.2.1 h1:n6EPaDyLSvCEa3frruQvAiHuNp2dhBlMSmkEr+HuzGc= +github.com/Masterminds/sprig/v3 v3.2.1/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= @@ -297,6 +303,7 @@ github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -336,6 +343,11 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/hashicorp/vault/api v1.14.0 h1:Ah3CFLixD5jmjusOgm8grfN9M0d+Y8fVR2SW0K6pJLU= github.com/hashicorp/vault/api v1.14.0/go.mod h1:pV9YLxBGSz+cItFDd8Ii4G17waWOQ32zVjMWHe/cOqk= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= +github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -400,6 +412,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/pkcs11 v1.0.2 h1:CIBkOawOtzJNE0B+EpRiUBzuVW7JEQAwdwhSS6YhIeg= github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= @@ -407,6 +421,8 @@ github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTS github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/swarmkit/v2 v2.0.0-20240611172349-ea1a7cec35cb h1:1UTTg2EgO3nuyV03wREDzldqqePzQ4+0a5G1C1y1bIo= @@ -496,6 +512,8 @@ github.com/samber/slog-gin v1.13.3 h1:BXVMDktx27zrr/PMYLvrEAOeIylBFtuemlQjgDUT3f github.com/samber/slog-gin v1.13.3/go.mod h1:7+YTBV20co5pQ+802hgAncESKtcZMAOKFUBpuT8IhXo= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -509,6 +527,7 @@ github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIK github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v0.0.0-20150508191742-4d07383ffe94/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v0.0.1/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= @@ -611,6 +630,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -755,6 +775,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/swarmcd/init.go b/swarmcd/init.go index e17940d..e54824b 100644 --- a/swarmcd/init.go +++ b/swarmcd/init.go @@ -14,9 +14,10 @@ import ( ) type StackStatus struct { - Error string - Revision string - RepoURL string + Error string + Revision string + RepoURL string + Templated bool } var config *util.Config = &util.Configs @@ -98,7 +99,7 @@ func initStacks() error { return fmt.Errorf("error initializing %s stack, no such repo: %s", stack, stackConfig.Repo) } discoverSecrets := config.SopsSecretsDiscovery || stackConfig.SopsSecretsDiscovery - swarmStack := newSwarmStack(stack, stackRepo, stackConfig.Branch, stackConfig.ComposeFile, stackConfig.SopsFiles, stackConfig.ValuesFile, discoverSecrets) + swarmStack := NewSwarmStack(stack, stackRepo, stackConfig.Branch, stackConfig.ComposeFile, stackConfig.SopsFiles, stackConfig.ValuesFile, discoverSecrets, config.GlobalValues, "template") stacks = append(stacks, swarmStack) stackStatus[stack] = &StackStatus{} stackStatus[stack].RepoURL = stackRepo.url diff --git a/swarmcd/repo.go b/swarmcd/repo.go index b3a3e19..98b02d9 100644 --- a/swarmcd/repo.go +++ b/swarmcd/repo.go @@ -12,25 +12,25 @@ import ( ) type stackRepo struct { - name string - lock *sync.Mutex - url string - gitRepoObject *git.Repository - auth *http.BasicAuth - path string + name string + lock *sync.Mutex + url string + gitRepoObject *git.Repository + auth *http.BasicAuth + path string } -func newStackRepo(name string, path string, url string, auth *http.BasicAuth) (*stackRepo, error) { +func newStackRepo(name string, repoPath string, url string, auth *http.BasicAuth) (*stackRepo, error) { var repo *git.Repository cloneOptions := &git.CloneOptions{ URL: url, Auth: auth, } - repo, err := git.PlainClone(path, false, cloneOptions) + repo, err := git.PlainClone(repoPath, false, cloneOptions) if err != nil { if errors.Is(err, git.ErrRepositoryAlreadyExists) { - repo, err = git.PlainOpen(path) + repo, err = git.PlainOpen(repoPath) if err != nil { return nil, fmt.Errorf("could not open existing repo %s: %w", name, err) } @@ -44,13 +44,14 @@ func newStackRepo(name string, path string, url string, auth *http.BasicAuth) (* return nil, fmt.Errorf("could not clone repo %s: %w", name, err) } } + return &stackRepo{ - name: name, - path: path, - url: url, - auth: auth, - lock: &sync.Mutex{}, - gitRepoObject: repo, + name: name, + path: repoPath, + url: url, + auth: auth, + lock: &sync.Mutex{}, + gitRepoObject: repo, }, nil } diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 3bc6464..b5442e8 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -5,13 +5,16 @@ import ( "crypto/md5" "fmt" "log/slog" + "maps" "os" "path" + "path/filepath" "text/template" "github.com/docker/cli/cli/command/stack" "github.com/goccy/go-yaml" "github.com/m-adawi/swarm-cd/util" + "github.com/Masterminds/sprig/v3" ) type swarmStack struct { @@ -22,9 +25,21 @@ type swarmStack struct { sopsFiles []string valuesFile string discoverSecrets bool + globalValuesMap map[string]any + templateFolder string + templated bool } -func newSwarmStack(name string, repo *stackRepo, branch string, composePath string, sopsFiles []string, valuesFile string, discoverSecrets bool) *swarmStack { +func NewSwarmStack(name string, repo *stackRepo, branch string, composePath string, sopsFiles []string, valuesFile string, discoverSecrets bool, globalValuesMap map[string]any, templateFolder string) *swarmStack { + if repo != nil { + templateFolder = path.Join(repo.path, templateFolder) + } + + _, err := os.Stat(templateFolder) + if err != nil { + templateFolder = "" + } + return &swarmStack{ name: name, repo: repo, @@ -33,6 +48,9 @@ func newSwarmStack(name string, repo *stackRepo, branch string, composePath stri sopsFiles: sopsFiles, valuesFile: valuesFile, discoverSecrets: discoverSecrets, + globalValuesMap: globalValuesMap, + templateFolder: templateFolder, + templated: false, } } @@ -41,7 +59,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { slog.String("stack", swarmStack.name), slog.String("branch", swarmStack.branch), ) - + log.Debug("pulling changes...") revision, err = swarmStack.repo.pullChanges(swarmStack.branch) if err != nil { @@ -49,19 +67,11 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("changes pulled", "revision", revision) - log.Debug("reading stack file...") - stackBytes, err := swarmStack.readStack() - if err != nil { - return - } - - if swarmStack.valuesFile != "" { - log.Debug("rendering template...") - stackBytes, err = swarmStack.renderComposeTemplate(stackBytes) - } + stackBytes, err := swarmStack.GenerateStack() if err != nil { return } + stackStatus[swarmStack.name].Templated = swarmStack.templated log.Debug("parsing stack content...") stackContents, err := swarmStack.parseStackString([]byte(stackBytes)) @@ -92,35 +102,85 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { return } -func (swarmStack *swarmStack) readStack() ([]byte, error) { - composeFile := path.Join(swarmStack.repo.path, swarmStack.composePath) - composeFileBytes, err := os.ReadFile(composeFile) + +func (swarmStack *swarmStack) GenerateStack() (stackBytes []byte, err error) { + log := logger.With( + slog.String("stack", swarmStack.name), + slog.String("branch", swarmStack.branch), + ) + log.Debug("reading stack file...") + swarmStack.templated = false + stackBytes, err = swarmStack.ReadStack() if err != nil { - return nil, fmt.Errorf("could not read compose file %s: %w", composeFile, err) + return } - return composeFileBytes, nil -} -func (swarmStack *swarmStack) renderComposeTemplate(templateContents []byte) ([]byte, error) { - valuesFile := path.Join(swarmStack.repo.path, swarmStack.valuesFile) - valuesBytes, err := os.ReadFile(valuesFile) - if err != nil { - return nil, fmt.Errorf("could not read %s stack values file: %w", swarmStack.name, err) + mergedValuesMap := make(map[string]any) + maps.Copy(mergedValuesMap, swarmStack.globalValuesMap) + + if swarmStack.valuesFile != "" { + valuesFile := swarmStack.valuesFile + if swarmStack.repo != nil { + valuesFile = path.Join(swarmStack.repo.path, swarmStack.valuesFile) + } + var valuesMap map[string]any + valuesMap, err = util.ParseValuesFile(valuesFile, swarmStack.name + " stack") + if err != nil { + return + } + maps.Copy(mergedValuesMap, valuesMap) } - var valuesMap map[string]any - yaml.Unmarshal(valuesBytes, &valuesMap) - templ, err := template.New(swarmStack.name).Parse(string(templateContents[:])) + + if len(mergedValuesMap) == 0 && swarmStack.templateFolder == "" { + // No need to continue, this file isn't templated + return + } + + log.Debug("rendering template...") + templ, err := template.New(swarmStack.name).Funcs(sprig.FuncMap()).Parse(string(stackBytes[:])) if err != nil { return nil, fmt.Errorf("could not parse %s stack compose file as a Go template: %w", swarmStack.name, err) } + + if swarmStack.templateFolder != "" { + log.Debug("Loading template folder...") + + pattern := path.Join(swarmStack.templateFolder, "*.tmpl") + filenames, err := filepath.Glob(pattern) + if err == nil { + if len(filenames) != 0 { + _, err = templ.ParseFiles(filenames...) + } else { + log.Debug("Skipping, folder empty", "folder", swarmStack.templateFolder) + } + } + if err != nil { + log.Error("Could not parse templates, trying to generate stack without them.", "error", err) + } + } + var stackContents bytes.Buffer - err = templ.Execute(&stackContents, map[string]map[string]any{"Values": valuesMap}) + err = templ.Execute(&stackContents, map[string]map[string]any{"Values": mergedValuesMap}) if err != nil { return nil, fmt.Errorf("error rending %s stack compose template: %w", swarmStack.name, err) } + // If there hasn't been any variable replacement, then it's not templated. + swarmStack.templated = !bytes.Equal(stackContents.Bytes(), stackBytes) return stackContents.Bytes(), nil } +func (swarmStack *swarmStack) ReadStack() ([]byte, error) { + composeFile := swarmStack.composePath + if swarmStack.repo != nil { + composeFile = path.Join(swarmStack.repo.path, swarmStack.composePath) + } + composeFileBytes, err := os.ReadFile(composeFile) + if err != nil { + return nil, fmt.Errorf("could not read compose file %s: %w", composeFile, err) + } + return composeFileBytes, nil +} + func (swarmStack *swarmStack) parseStackString(stackContent []byte) (map[string]any, error) { var composeMap map[string]any err := yaml.Unmarshal(stackContent, &composeMap) diff --git a/swarmcd/stack_test.go b/swarmcd/stack_test.go index 59de84a..7380b89 100644 --- a/swarmcd/stack_test.go +++ b/swarmcd/stack_test.go @@ -8,7 +8,8 @@ import ( // External objects are ignored by the rotation func TestRotateExternalObjects(t *testing.T) { repo := &stackRepo{name: "test", path: "test", url: "", auth: nil, lock: &sync.Mutex{}, gitRepoObject: nil} - stack := newSwarmStack("test", repo, "main", "docker-compose.yaml", nil, "", false) + var valuesMap map[string]any + stack := NewSwarmStack("test", repo, "main", "docker-compose.yaml", nil, "", false, valuesMap, "template") objects := map[string]any{ "my-secret": map[string]any{"external": true}, } @@ -21,7 +22,8 @@ func TestRotateExternalObjects(t *testing.T) { // Secrets are discovered, external secrets are ignored func TestSecretDiscovery(t *testing.T) { repo := &stackRepo{name: "test", path: "test", url: "", auth: nil, lock: &sync.Mutex{}, gitRepoObject: nil} - stack := newSwarmStack("test", repo, "main", "stacks/docker-compose.yaml", nil, "", false) + var valuesMap map[string]any + stack := NewSwarmStack("test", repo, "main", "stacks/docker-compose.yaml", nil, "", false, valuesMap, "template") stackString := []byte(`services: my-service: image: my-image diff --git a/swarmcd/swarmcd.go b/swarmcd/swarmcd.go index 0e6da95..a77be22 100644 --- a/swarmcd/swarmcd.go +++ b/swarmcd/swarmcd.go @@ -46,3 +46,12 @@ func updateStackThread(swarmStack *swarmStack, waitGroup *sync.WaitGroup) { func GetStackStatus() map[string]*StackStatus { return stackStatus } + +func GetSwarmStack(stackName string) (*swarmStack, error) { + for _, elem := range stacks { + if elem.name == stackName { + return elem, nil + } + } + return nil, fmt.Errorf("Stack %s does not exist.", stackName) +} diff --git a/ui/src/components/StatusCard.tsx b/ui/src/components/StatusCard.tsx index bc78df6..a179489 100644 --- a/ui/src/components/StatusCard.tsx +++ b/ui/src/components/StatusCard.tsx @@ -5,12 +5,14 @@ function StatusCard({ name, error, revision, - repoURL + repoURL, + templated }: Readonly<{ name: string error: string revision: string repoURL: string + templated: boolean }>): React.ReactElement { return ( @@ -32,6 +34,19 @@ function StatusCard({ {repoURL} + + Compose file: + + compose.yaml + + {templated && ( + <> + Rendered file: + + rendered.yaml + + + )} ) diff --git a/ui/src/components/StatusCardList.tsx b/ui/src/components/StatusCardList.tsx index 4f87ba0..444f0c1 100644 --- a/ui/src/components/StatusCardList.tsx +++ b/ui/src/components/StatusCardList.tsx @@ -21,7 +21,7 @@ function StatusCardList({ statuses, query }: Readonly<{ statuses: StackStatus[]; ) : ( filteredStatuses.map((item, index) => ( - + )) )} diff --git a/ui/src/hooks/dummyStackStatuses.json b/ui/src/hooks/dummyStackStatuses.json index b3e95fc..908835f 100644 --- a/ui/src/hooks/dummyStackStatuses.json +++ b/ui/src/hooks/dummyStackStatuses.json @@ -3,18 +3,21 @@ "Name": "Project Alpha", "Error": "", "Revision": "v1.0.1", - "RepoURL": "https://github.com/user/project-alpha" + "RepoURL": "https://github.com/user/project-alpha", + "Templated": false }, { "Name": "Project Beta", "Error": "Failed to build", "Revision": "v2.3.4", - "RepoURL": "https://github.com/user/project-beta" + "RepoURL": "https://github.com/user/project-beta", + "Templated": false }, { "Name": "Project Gamma", "Error": "", "Revision": "v0.9.8", - "RepoURL": "https://github.com/user/project-gamma" + "RepoURL": "https://github.com/user/project-gamma", + "Templated": false } ] diff --git a/ui/src/hooks/useFetchStatuses.tsx b/ui/src/hooks/useFetchStatuses.tsx index b1b2b03..4952a2b 100644 --- a/ui/src/hooks/useFetchStatuses.tsx +++ b/ui/src/hooks/useFetchStatuses.tsx @@ -6,6 +6,7 @@ export interface StackStatus { Error: string Revision: string RepoURL: string + Templated: boolean } async function fetchFromServer(): Promise { diff --git a/ui/tests/components/StatusCard.test.tsx b/ui/tests/components/StatusCard.test.tsx index c2e4703..be9eb51 100644 --- a/ui/tests/components/StatusCard.test.tsx +++ b/ui/tests/components/StatusCard.test.tsx @@ -4,8 +4,8 @@ import StatusCard from "../../src/components/StatusCard" describe("StatusCard", () => { const status = { name: "Some Name Here", revision: "3.76.1", repoURL: "https://www.github.com/1234" } - it("should render name, revision, and repoURL properties", () => { - render() + it("should render name, revision, repoURL and compose properties", () => { + render() for (const value of Object.values(status)) { const valueElement = screen.getByText(new RegExp(value, "i")) @@ -14,24 +14,40 @@ describe("StatusCard", () => { }) it("should render repoURL as a link", () => { - render() + render() const repoUrlElement = screen.getByRole("link", { name: status.repoURL }) expect(repoUrlElement).toBeInTheDocument() expect(repoUrlElement).toHaveAttribute("href", status.repoURL) }) + + it("should render compose as a link", () => { + render() + + const repoUrlElement = screen.getByRole("link", { name: "compose.yaml" }) + expect(repoUrlElement).toBeInTheDocument() + expect(repoUrlElement).toHaveAttribute("href", "/stacks/" + status.name + "/compose.yaml") + }) it("should not render error if it is empty", () => { - render() + render() const errorText = screen.queryByText(/error/i) expect(errorText).not.toBeInTheDocument() }) it("should render error if it is not empty", () => { - render() + render() const errorText = screen.queryByText(/error/i) expect(errorText).toBeInTheDocument() }) + + it("should show a rendered path if it is templated", () => { + render() + + const templatePathElement = screen.getByRole("link", { name: "rendered.yaml" }) + expect(templatePathElement).toBeInTheDocument() + expect(templatePathElement).toHaveAttribute("href", "/stacks/" + status.name + "/rendered.yaml") + }) }) diff --git a/ui/tests/components/StatusCardList.test.tsx b/ui/tests/components/StatusCardList.test.tsx index 77a1769..526db43 100644 --- a/ui/tests/components/StatusCardList.test.tsx +++ b/ui/tests/components/StatusCardList.test.tsx @@ -4,9 +4,9 @@ import { StackStatus } from "../../src/hooks/useFetchStatuses" describe("StatusCardList", () => { const statuses: StackStatus[] = [ - { Name: "Foobar", Error: "", Revision: "1.0.0", RepoURL: "https://www.url1.com" }, - { Name: "FooFoo", Error: "", Revision: "2.0.0", RepoURL: "https://www.url2.com" }, - { Name: "Boobaz", Error: "Oh no!!!", Revision: "2.0.0", RepoURL: "https://www.url3.com" } + { Name: "Foobar", Error: "", Revision: "1.0.0", RepoURL: "https://www.url1.com", Templated: false }, + { Name: "FooFoo", Error: "", Revision: "2.0.0", RepoURL: "https://www.url2.com", Templated: false }, + { Name: "Boobaz", Error: "Oh no!!!", Revision: "2.0.0", RepoURL: "https://www.url3.com", Templated: false } ] it("should render no statuses if the list of statuses is empty", () => { diff --git a/util/config.go b/util/config.go index 174ced2..9e340dd 100644 --- a/util/config.go +++ b/util/config.go @@ -3,6 +3,7 @@ package util import ( "errors" "fmt" + "os" "github.com/spf13/viper" ) @@ -31,12 +32,13 @@ type Config struct { RepoConfigs map[string]*RepoConfig `mapstructure:"repos"` SopsSecretsDiscovery bool `mapstructure:"sops_secrets_discovery"` Address string `mapstructure:"address"` + GlobalValues map[string]any `mapstructure:"global_values"` } var Configs Config func LoadConfigs() (err error) { - err = readConfig() + err = ReadConfig("") if err != nil { return fmt.Errorf("could not read configuration file: %w", err) } @@ -52,13 +54,24 @@ func LoadConfigs() (err error) { return fmt.Errorf("could not load stacks file: %w", err) } } + if Configs.GlobalValues == nil { + if _, errStat := os.Stat("global_values.yaml"); errStat == nil { + Configs.GlobalValues, err = ParseValuesFile("global_values.yaml", "global") + if err != nil { + return + } + } + } return } -func readConfig() (err error) { +func ReadConfig(configPath string) (err error) { configViper := viper.New() configViper.SetConfigName("config") configViper.AddConfigPath(".") + if configPath != "" { + configViper.SetConfigFile(configPath) + } configViper.SetDefault("update_interval", 120) configViper.SetDefault("repos_path", "repos") configViper.SetDefault("auto_rotate", true) diff --git a/util/values_map.go b/util/values_map.go new file mode 100644 index 0000000..7394c56 --- /dev/null +++ b/util/values_map.go @@ -0,0 +1,17 @@ +package util + +import ( + "fmt" + "os" + "github.com/goccy/go-yaml" +) + +func ParseValuesFile(valuesFile string, source string) (map[string]any, error){ + valuesBytes, err := os.ReadFile(valuesFile) + if err != nil { + return nil, fmt.Errorf("could not read %s values file: %w", source, err) + } + var valuesMap map[string]any + yaml.Unmarshal(valuesBytes, &valuesMap) + return valuesMap, nil +} diff --git a/web/controllers.go b/web/controllers.go index 14a4721..76ad634 100644 --- a/web/controllers.go +++ b/web/controllers.go @@ -10,17 +10,46 @@ import ( func getStacks(ctx *gin.Context) { stacksStatus := swarmcd.GetStackStatus() - var stacks []map[string]string + var stacks []map[string]any for k, v := range stacksStatus { - stacks = append(stacks, map[string]string{ + stacks = append(stacks, map[string]any{ "Name": k, "Error": v.Error, "RepoURL": v.RepoURL, "Revision": v.Revision, + "Templated": v.Templated, }) } sort.Slice(stacks, func(i, j int) bool { - return stacks[i]["Name"] < stacks[j]["Name"] + return stacks[i]["Name"].(string) < stacks[j]["Name"].(string) }) ctx.JSON(http.StatusOK, stacks) -} \ No newline at end of file +} + +func getCompose(ctx *gin.Context) { + stackName := ctx.Param("stackName") + swarmStack, err := swarmcd.GetSwarmStack(stackName) + if err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + } + + stackBytes, err := swarmStack.ReadStack() + if err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + } + ctx.String(200, string(stackBytes)) +} + +func getRendered(ctx *gin.Context) { + stackName := ctx.Param("stackName") + swarmStack, err := swarmcd.GetSwarmStack(stackName) + if err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + } + + stackBytes, err := swarmStack.GenerateStack() + if err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + } + ctx.String(200, string(stackBytes)) +} diff --git a/web/routes.go b/web/routes.go index 713946a..af3dbaa 100644 --- a/web/routes.go +++ b/web/routes.go @@ -12,6 +12,8 @@ var router *gin.Engine = gin.New() func init() { router.Use(sloggin.New(util.Logger)) router.GET("/stacks", getStacks) + router.GET("/stacks/:stackName/compose.yaml", getCompose) + router.GET("/stacks/:stackName/rendered.yaml", getRendered) router.StaticFile("/ui", "ui/index.html") router.Static("/assets", "ui/assets") router.GET("/", func(c *gin.Context) {