add split command and git auth

Author: schlapzz

.vscode/launch.json

@@ -4,13 +4,21 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "Split",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "main.go",
+            "args": ["split", "/home/schlatter/tmp/argo-apps/out/all.yaml", "--output-dir=out/split"]
+        },
         {
             "name": "ArgoCD",
             "type": "go",
             "request": "launch",
             "mode": "auto",
             "program": "main.go",
-            "args": ["argocd", "./testdata/argocd/", "--repoServer=localhost:8081", "--output-dir=out/"]
+            "args": ["argocd", "./out/split", "--repoServer=localhost:8081", "--output-dir=out/",  "-u", "argo", "-i", "/home/schlatter/.ssh/id_rsa"]
         },
         {
             "name": "Kustomize build",

argocd/repoClient.go

@@ -3,6 +3,7 @@ package argocd
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"goff/util"
 	"io/fs"
 	"os"
@@ -16,9 +17,15 @@ import (
 	"github.com/ghodss/yaml"
 )
 
-func Render(dir, repoServerUrl, outputDir string) {
-	conn := apiclient.NewRepoServerClientset(repoServerUrl, 300, apiclient.TLSConfiguration{StrictValidation: false})
-	r, b, err := conn.NewRepoServerClient()
+type RepoCredentails struct {
+	Username string
+	Password string
+	KeyFile  string
+}
+
+func Render(dir, repoServerUrl, outputDir string, creds RepoCredentails) {
+	conn := apiclient.NewRepoServerClientset(repoServerUrl, 600, apiclient.TLSConfiguration{StrictValidation: false})
+	r, client, err := conn.NewRepoServerClient()
 	defer r.Close()
 
 	if err != nil {
@@ -32,93 +39,124 @@ func Render(dir, repoServerUrl, outputDir string) {
 	}
 
 	for _, file := range files {
-		renderFile(file, repoServerUrl, outputDir, b)
+		err = renderFile(file, repoServerUrl, outputDir, client, creds)
+		if err != nil {
+			fmt.Println(err)
+		}
+		//TODO log errors
 	}
 
 }
 
-func renderFile(file, repoServerUrl, outputDir string, client apiclient.RepoServerServiceClient) {
+func renderFile(file, repoServerUrl, outputDir string, client apiclient.RepoServerServiceClient, creds RepoCredentails) error {
 
 	data, err := os.ReadFile(file)
 	if err != nil {
-		panic(err)
+		return err
 	}
 
 	data, err = yaml.YAMLToJSON(data)
 	if err != nil {
-		panic(err)
+		return err
 	}
 
 	app := &v1alpha1.Application{}
 
 	err = json.Unmarshal(data, app)
 	if err != nil {
-		panic(err)
+		return err
 	}
 
 	repoDB := &dbmocks.ArgoDB{}
+	source := v1alpha1.ApplicationSource{}
+
+	var privateKey string
+	if creds.KeyFile != "" {
+		data, err := os.ReadFile(creds.KeyFile)
+		if err != nil {
+			return err
+		}
+		privateKey = string(data)
+	}
 
 	if app.Spec.Source != nil {
 		repoDB.On("GetRepository", context.Background(), app.Spec.Source.RepoURL).Return(&v1alpha1.Repository{
-			Repo: app.Spec.Source.RepoURL,
+			Repo:               app.Spec.Source.RepoURL,
+			SSHPrivateKey:      privateKey,
+			Username:           creds.Username,
+			Password:           creds.Password,
+			ForceHttpBasicAuth: true,
 		}, nil)
+		source = *app.Spec.Source
 	}
 
 	if app.Spec.Sources != nil {
 		for i := range app.Spec.Sources {
+			source = app.Spec.Sources[i]
 			repo := app.Spec.Sources[i].RepoURL
 			if repo != "" {
 				repoDB.On("GetRepository", context.Background(), repo).Return(&v1alpha1.Repository{
-					Repo: repo,
+					Repo:               repo,
+					SSHPrivateKey:      privateKey,
+					Username:           creds.Username,
+					Password:           creds.Password,
+					ForceHttpBasicAuth: true,
 				}, nil)
 			}
 		}
 	}
 
 	refSources, err := argo.GetRefSources(context.Background(), app.Spec, repoDB)
 	req := &apiclient.ManifestRequest{
-		ApplicationSource:  &app.Spec.Sources[0],
+		ApplicationSource:  &source,
 		AppName:            "goff-test",
 		NoCache:            true,
 		RefSources:         refSources,
 		HasMultipleSources: true,
-		Revision:           app.Spec.Sources[0].TargetRevision,
+		Revision:           source.TargetRevision,
+		KustomizeOptions: &v1alpha1.KustomizeOptions{
+			BuildOptions: "--enable-helm",
+		},
 		Repo: &v1alpha1.Repository{
-			Repo: app.Spec.Sources[0].RepoURL,
+			Repo:               source.RepoURL,
+			SSHPrivateKey:      privateKey,
+			Username:           creds.Username,
+			Password:           creds.Password,
+			ForceHttpBasicAuth: true,
 		},
 	}
 
 	resp, err := client.GenerateManifest(context.Background(), req)
 	if err != nil {
-		panic(err)
+		return fmt.Errorf("could not process application '%s': %w", app.Name, err)
 	}
 
 	err = os.MkdirAll(outputDir, 0777)
 	if err != nil {
-		panic(err)
+		return err
 	}
 
 	for _, manifest := range resp.Manifests {
 
 		fileName, err := util.FileNameFromManifest(manifest)
 		if err != nil {
-			panic(err)
+			return err
 		}
 
 		outputFile := filepath.Join(outputDir, fileName)
 
 		yamlManifest, err := yaml.JSONToYAML([]byte(manifest))
 		if err != nil {
-			panic(err)
+			return err
 		}
 
 		err = os.WriteFile(outputFile, yamlManifest, 0777)
 		if err != nil {
-			panic(err)
+			return err
 		}
 
 	}
-
+	return nil
 }
 
 func findArgoApps(rootDir string) ([]string, error) {

cmd/argocd.go

@@ -13,6 +13,9 @@ import (
 var repoServerUrl *string
 
 var argoOutputDir *string
+var repoUsername *string
+var repoPassword *string
+var repoSshKey *string
 
 // argocdCmd represents the argocd command
 var argocdCmd = &cobra.Command{
@@ -21,21 +24,20 @@ var argocdCmd = &cobra.Command{
 	Args:  cobra.ExactArgs(1),
 	Long:  `Render manifests from ArgoCD Application`,
 	Run: func(cmd *cobra.Command, args []string) {
-		argocd.Render(args[0], *repoServerUrl, *argoOutputDir)
+		argocd.Render(args[0], *repoServerUrl, *argoOutputDir, argocd.RepoCredentails{
+			Username: *repoUsername,
+			Password: *repoPassword,
+			KeyFile:  *repoSshKey,
+		})
 	},
 }
 
 func init() {
 	rootCmd.AddCommand(argocdCmd)
 
-	// Here you will define your flags and configuration settings.
-
-	// Cobra supports Persistent Flags which will work for this command
-	// and all subcommands, e.g.:
-	// argocdCmd.PersistentFlags().String("foo", "", "A help for foo")
-
-	// Cobra supports local flags which will only run when this command
-	// is called directly, e.g.:
 	repoServerUrl = argocdCmd.Flags().String("repoServer", "localhost:8081", "URL to argoCD repo server")
 	argoOutputDir = argocdCmd.Flags().StringP("output-dir", "o", ".", "Output directory")
+	repoUsername = argocdCmd.Flags().StringP("username", "u", "", "Repo username")
+	repoPassword = argocdCmd.Flags().StringP("password", "p", "", "Repo password")
+	repoSshKey = argocdCmd.Flags().StringP("ssh-key", "i", "", "Repo SSH Key")
 }

cmd/split.go

@@ -0,0 +1,32 @@
+/*
+Copyright © 2023 NAME HERE <EMAIL ADDRESS>
+
+*/
+package cmd
+
+import (
+	"goff/util"
+
+	"github.com/spf13/cobra"
+)
+
+var outputSplitDir *string
+
+// diffCmd represents the diff command
+var splitCmd = &cobra.Command{
+	Use:   "split",
+	Short: "Split manifests [manifestFile]",
+	Args:  cobra.ExactArgs(1),
+	Long:  `Split multi document yaml`,
+	Run: func(cmd *cobra.Command, args []string) {
+		err := util.SplitManifests(args[0], *outputSplitDir)
+		if err != nil {
+			panic(err)
+		}
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(splitCmd)
+	outputSplitDir = splitCmd.Flags().StringP("output-dir", "o", ".", "Output directory")
+}

docker-compose.yml

@@ -0,0 +1,14 @@
+version: "3.9"
+services:
+  redis:
+    image: "redis"
+    ports:
+      - "6379:6379"
+  repos:
+    image: "quay.io/argoproj/argocd:latest"
+    command:
+     - "--redis"
+     - "redis:6379"
+    ports:
+      - "8081:8081"
+    entrypoint: argocd-repo-server

util/util.go

@@ -1,11 +1,54 @@
 package util
 
 import (
+	"bytes"
 	"fmt"
+	"os"
+	"path/filepath"
 
 	"github.com/ghodss/yaml"
 )
 
+//Multi doc yaml, split and save
+func SplitManifests(manifestFile, outDir string) error {
+
+	data, err := os.ReadFile(manifestFile)
+	if err != nil {
+		return err
+	}
+
+	err = os.MkdirAll(outDir, 0777)
+	if err != nil {
+		return err
+	}
+
+	splitted := bytes.Split(data, []byte("---"))
+
+	for i := range splitted {
+		if len(splitted[i]) == 0 {
+			continue
+		}
+
+		res := &Ressource{}
+		err := yaml.Unmarshal(splitted[i], res)
+		if err != nil {
+			return err
+		}
+
+		filename := fmt.Sprintf("%s-%s.yaml", res.Kind, res.Metadata.Name)
+		filename = filepath.Join(outDir, filename)
+
+		err = os.WriteFile(filename, []byte(splitted[i]), 0777)
+		if err != nil {
+			return err
+		}
+		fmt.Println("wrote file at: " + filename)
+	}
+
+	return nil
+
+}
+
 func FileNameFromManifest(manifest string) (string, error) {
 	res := &Ressource{}
 	err := yaml.Unmarshal([]byte(manifest), res)