feat: add env variable support (#209)

Spartakovic · web-flow · commit 6185537602e6 · 2023-05-09T16:14:04.000+01:00
* feat: add env variable support
diff --git a/action.yml b/action.yml
@@ -65,16 +65,6 @@ outputs:
 runs:
   using: composite
   steps:
-    - name: Validate General Input Configuration
-      run: |
-        if [[ "${{ inputs.setup-aws }}" == "false" && "${{ inputs.setup-google-cloud }}" == "false" ]]; then
-          echo "Either 'setup-aws' or 'setup-google-cloud' input must be set to 'true'"
-        else
-          exit 0
-        fi
-        exit 1
-      shell: bash
-
     - name: Validate Input Configuration for Google
       run: |
         if [[ -z ${{ toJSON(inputs.google-auth-credentials) }} && -z "${{ inputs.google-workload-identity-provider }}" ]]; then
diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go
@@ -25,6 +25,12 @@ type DiggerConfigYaml struct {
 	GenerateProjectsConfig *GenerateProjectsConfig `yaml:"generate_projects"`
 }
 
+type EnvVarConfig struct {
+	Name      string `yaml:"name"`
+	ValueFrom string `yaml:"value_from"`
+	Value     string `yaml:"value"`
+}
+
 type DiggerConfig struct {
 	Projects         []Project
 	AutoMerge        bool
@@ -50,11 +56,17 @@ type Stage struct {
 }
 
 type Workflow struct {
+	EnvVars       EnvVars                `yaml:"env_vars"`
 	Plan          *Stage                 `yaml:"plan,omitempty"`
 	Apply         *Stage                 `yaml:"apply,omitempty"`
 	Configuration *WorkflowConfiguration `yaml:"workflow_configuration"`
 }
 
+type EnvVars struct {
+	State    []EnvVarConfig `yaml:"state"`
+	Commands []EnvVarConfig `yaml:"commands"`
+}
+
 type DirWalker interface {
 	GetDirs(workingDir string) ([]string, error)
 }
@@ -125,6 +137,10 @@ func (w *Workflow) UnmarshalYAML(unmarshal func(interface{}) error) error {
 				},
 			},
 		},
+		EnvVars: EnvVars{
+			State:    []EnvVarConfig{},
+			Commands: []EnvVarConfig{},
+		},
 	}
 	if err := unmarshal(&raw); err != nil {
 		return err
diff --git a/pkg/configuration/digger_config_test.go b/pkg/configuration/digger_config_test.go
@@ -116,6 +116,56 @@ workflows:
 	assert.Equal(t, Step{Action: "run", Value: "echo \"hello\""}, dg.Workflows["myworkflow"].Plan.Steps[0], "parsed struct does not match expected struct")
 }
 
+func TestEnvVarsConfiguration(t *testing.T) {
+	tempDir, teardown := setUp()
+	defer teardown()
+
+	diggerCfg := `
+projects:
+- name: dev
+  branch: /main/
+  dir: .
+  workspace: default
+  terragrunt: false
+  workflow: myworkflow
+workflows:
+  myworkflow:
+    plan:
+      steps:
+      - init:
+          extra_args: ["-lock=false"]
+      - plan:
+          extra_args: ["-lock=false"]
+      - run: echo "hello"
+    apply:
+      steps:
+      - apply:
+          extra_args: ["-lock=false"]
+    workflow_configuration:
+      on_pull_request_pushed: [digger plan]
+      on_pull_request_closed: [digger unlock]
+      on_commit_to_default: [digger apply]
+    env_vars:
+      state:
+      - name: TF_VAR_state
+        value: s3://mybucket/terraform.tfstate
+      commands:
+      - name: TF_VAR_command
+        value: plan
+`
+	deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg)
+	defer deleteFile()
+
+	dg, err := NewDiggerConfig(tempDir, &FileSystemDirWalker{})
+	assert.NoError(t, err, "expected error to be nil")
+	assert.Equal(t, []EnvVarConfig{
+		{Name: "TF_VAR_state", Value: "s3://mybucket/terraform.tfstate"},
+	}, dg.Workflows["myworkflow"].EnvVars.State, "parsed struct does not match expected struct")
+	assert.Equal(t, []EnvVarConfig{
+		{Name: "TF_VAR_command", Value: "plan"},
+	}, dg.Workflows["myworkflow"].EnvVars.Commands, "parsed struct does not match expected struct")
+}
+
 func TestDefaultValuesForWorkflowConfiguration(t *testing.T) {
 	tempDir, teardown := setUp()
 	defer teardown()
diff --git a/pkg/digger/digger.go b/pkg/digger/digger.go
@@ -81,9 +81,10 @@ func RunCommandsPerProject(commandsPerProject []ProjectCommand, repoOwner string
 			}
 
 			commandRunner := CommandRunner{}
-
 			diggerExecutor := DiggerExecutor{
 				projectCommands.ProjectName,
+				projectCommands.StateEnvVars,
+				projectCommands.CommandEnvVars,
 				projectCommands.ApplyStage,
 				projectCommands.PlanStage,
 				commandRunner,
@@ -186,6 +187,8 @@ type ProjectCommand struct {
 	Commands         []string
 	ApplyStage       *configuration.Stage
 	PlanStage        *configuration.Stage
+	StateEnvVars     map[string]string
+	CommandEnvVars   map[string]string
 }
 
 func ConvertGithubEventToCommands(event models.Event, impactedProjects []configuration.Project, workflows map[string]configuration.Workflow) ([]ProjectCommand, error) {
@@ -199,6 +202,9 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu
 			if !ok {
 				workflow = *defaultWorkflow()
 			}
+
+			stateEnvVars, commandEnvVars := collectEnvVars(workflow.EnvVars)
+
 			if event.Action == "closed" && event.PullRequest.Merged && event.PullRequest.Base.Ref == event.Repository.DefaultBranch {
 				commandsPerProject = append(commandsPerProject, ProjectCommand{
 					ProjectName:      project.Name,
@@ -208,6 +214,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu
 					Commands:         workflow.Configuration.OnCommitToDefault,
 					ApplyStage:       workflow.Apply,
 					PlanStage:        workflow.Plan,
+					CommandEnvVars:   commandEnvVars,
+					StateEnvVars:     stateEnvVars,
 				})
 			} else if event.Action == "opened" || event.Action == "reopened" || event.Action == "synchronize" {
 				commandsPerProject = append(commandsPerProject, ProjectCommand{
@@ -218,6 +226,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu
 					Commands:         workflow.Configuration.OnPullRequestPushed,
 					ApplyStage:       workflow.Apply,
 					PlanStage:        workflow.Plan,
+					CommandEnvVars:   commandEnvVars,
+					StateEnvVars:     stateEnvVars,
 				})
 			} else if event.Action == "closed" {
 				commandsPerProject = append(commandsPerProject, ProjectCommand{
@@ -228,6 +238,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu
 					Commands:         workflow.Configuration.OnPullRequestClosed,
 					ApplyStage:       workflow.Apply,
 					PlanStage:        workflow.Plan,
+					CommandEnvVars:   commandEnvVars,
+					StateEnvVars:     stateEnvVars,
 				})
 			}
 		}
@@ -243,6 +255,9 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu
 					if !ok {
 						workflow = *defaultWorkflow()
 					}
+
+					stateEnvVars, commandEnvVars := collectEnvVars(workflow.EnvVars)
+
 					workspace := project.Workspace
 					workspaceOverride, err := parseWorkspace(event.Comment.Body)
 					if err != nil {
@@ -259,6 +274,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu
 						Commands:         []string{command},
 						ApplyStage:       workflow.Apply,
 						PlanStage:        workflow.Plan,
+						CommandEnvVars:   commandEnvVars,
+						StateEnvVars:     stateEnvVars,
 					})
 				}
 			}
@@ -269,6 +286,29 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu
 	}
 }
 
+func collectEnvVars(envs configuration.EnvVars) (map[string]string, map[string]string) {
+	stateEnvVars := map[string]string{}
+
+	for _, envvar := range envs.State {
+		if envvar.Value != "" {
+			stateEnvVars[envvar.Name] = envvar.Value
+		} else if envvar.ValueFrom != "" {
+			stateEnvVars[envvar.Name] = os.Getenv(envvar.ValueFrom)
+		}
+	}
+
+	commandEnvVars := map[string]string{}
+
+	for _, envvar := range envs.Commands {
+		if envvar.Value != "" {
+			commandEnvVars[envvar.Name] = envvar.Value
+		} else if envvar.ValueFrom != "" {
+			commandEnvVars[envvar.Name] = os.Getenv(envvar.ValueFrom)
+		}
+	}
+	return stateEnvVars, commandEnvVars
+}
+
 func parseWorkspace(comment string) (string, error) {
 	re := regexp.MustCompile(`-w(?:\s+(\S+)|$)`)
 	matches := re.FindAllStringSubmatch(comment, -1)
@@ -299,6 +339,8 @@ func parseProjectName(comment string) string {
 
 type DiggerExecutor struct {
 	projectName       string
+	stateEnvVars      map[string]string
+	commandEnvVars    map[string]string
 	applyStage        *configuration.Stage
 	planStage         *configuration.Stage
 	commandRunner     CommandRun
@@ -363,15 +405,15 @@ func (d DiggerExecutor) Plan(prNumber int) error {
 		}
 		for _, step := range planSteps {
 			if step.Action == "init" {
-				_, _, err := d.terraformExecutor.Init(step.ExtraArgs)
+				_, _, err := d.terraformExecutor.Init(step.ExtraArgs, d.stateEnvVars)
 				if err != nil {
 					return fmt.Errorf("error running init: %v", err)
 				}
 			}
 			if step.Action == "plan" {
 				planArgs := []string{"-out", d.planFileName()}
 				planArgs = append(planArgs, step.ExtraArgs...)
-				isNonEmptyPlan, stdout, stderr, err := d.terraformExecutor.Plan(planArgs)
+				isNonEmptyPlan, stdout, stderr, err := d.terraformExecutor.Plan(planArgs, d.commandEnvVars)
 				if err != nil {
 					return fmt.Errorf("error executing plan: %v", err)
 				}
@@ -435,13 +477,13 @@ func (d DiggerExecutor) Apply(prNumber int) error {
 
 			for _, step := range applySteps {
 				if step.Action == "init" {
-					_, _, err := d.terraformExecutor.Init(step.ExtraArgs)
+					_, _, err := d.terraformExecutor.Init(step.ExtraArgs, d.stateEnvVars)
 					if err != nil {
 						return fmt.Errorf("error running init: %v", err)
 					}
 				}
 				if step.Action == "apply" {
-					stdout, stderr, err := d.terraformExecutor.Apply(step.ExtraArgs, plansFilename)
+					stdout, stderr, err := d.terraformExecutor.Apply(step.ExtraArgs, plansFilename, d.commandEnvVars)
 					applyOutput := cleanupTerraformApply(true, err, stdout, stderr)
 					comment := utils.GetTerraformOutputAsCollapsibleComment("Apply for **"+d.lock.LockId()+"**", applyOutput)
 					d.prManager.PublishComment(prNumber, comment)
diff --git a/pkg/digger/digger_test.go b/pkg/digger/digger_test.go
@@ -30,20 +30,20 @@ type MockTerraformExecutor struct {
 	Commands []RunInfo
 }
 
-func (m *MockTerraformExecutor) Init(params []string) (string, string, error) {
+func (m *MockTerraformExecutor) Init(params []string, envs map[string]string) (string, string, error) {
 	m.Commands = append(m.Commands, RunInfo{"Init", strings.Join(params, " "), time.Now()})
 	return "", "", nil
 }
 
-func (m *MockTerraformExecutor) Apply(params []string, plan *string) (string, string, error) {
+func (m *MockTerraformExecutor) Apply(params []string, plan *string, envs map[string]string) (string, string, error) {
 	if plan != nil {
 		params = append(params, *plan)
 	}
 	m.Commands = append(m.Commands, RunInfo{"Apply", strings.Join(params, " "), time.Now()})
 	return "", "", nil
 }
 
-func (m *MockTerraformExecutor) Plan(params []string) (bool, string, string, error) {
+func (m *MockTerraformExecutor) Plan(params []string, envs map[string]string) (bool, string, string, error) {
 	m.Commands = append(m.Commands, RunInfo{"Plan", strings.Join(params, " "), time.Now()})
 	return true, "", "", nil
 }
diff --git a/pkg/integration/integration_test.go b/pkg/integration/integration_test.go
@@ -388,7 +388,7 @@ func TestHappyPath(t *testing.T) {
 	//  new pr should lock the project
 	impactedProjects, prNumber, _, err := digger.ProcessGitHubEvent(ghEvent, diggerConfig, githubPrService)
 	assert.NoError(t, err)
-	commandsToRunPerProject, err := digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, map[string]configuration.Workflow{})
+	commandsToRunPerProject, err := digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, diggerConfig.Workflows)
 	assert.NoError(t, err)
 	zipManager := utils.Zipper{}
 	planStorage := &utils.GithubPlanStorage{
@@ -423,7 +423,7 @@ func TestHappyPath(t *testing.T) {
 	// 'digger plan' comment should trigger terraform execution
 	impactedProjects, prNumber, _, err = digger.ProcessGitHubEvent(ghEvent, diggerConfig, githubPrService)
 	assert.NoError(t, err)
-	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, map[string]configuration.Workflow{})
+	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, diggerConfig.Workflows)
 	assert.NoError(t, err)
 	_, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, lock, planStorage, dir)
 	assert.NoError(t, err)
@@ -437,7 +437,7 @@ func TestHappyPath(t *testing.T) {
 	// 'digger apply' comment should trigger terraform execution and unlock the project
 	impactedProjects, prNumber, _, err = digger.ProcessGitHubEvent(ghEvent, diggerConfig, githubPrService)
 	assert.NoError(t, err)
-	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, map[string]configuration.Workflow{})
+	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, diggerConfig.Workflows)
 	assert.NoError(t, err)
 	_, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, lock, planStorage, dir)
 	assert.NoError(t, err)
@@ -461,7 +461,7 @@ func TestHappyPath(t *testing.T) {
 
 	impactedProjects, prNumber, _, err = digger.ProcessGitHubEvent(ghEvent, diggerConfig, githubPrService)
 	assert.NoError(t, err)
-	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, map[string]configuration.Workflow{})
+	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, diggerConfig.Workflows)
 	assert.NoError(t, err)
 	_, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, lock, planStorage, dir)
 	assert.NoError(t, err)
@@ -539,7 +539,7 @@ func TestMultiEnvHappyPath(t *testing.T) {
 	// no files changed, no locks
 	impactedProjects, prNumber, _, err := digger.ProcessGitHubEvent(ghEvent, diggerConfig, githubPrService)
 	assert.NoError(t, err)
-	commandsToRunPerProject, err := digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, map[string]configuration.Workflow{})
+	commandsToRunPerProject, err := digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, diggerConfig.Workflows)
 	assert.NoError(t, err)
 
 	zipManager := utils.Zipper{}
@@ -574,7 +574,7 @@ func TestMultiEnvHappyPath(t *testing.T) {
 	// 'digger plan' comment should trigger terraform execution
 	impactedProjects, prNumber, _, err = digger.ProcessGitHubEvent(ghEvent, diggerConfig, githubPrService)
 	assert.NoError(t, err)
-	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, map[string]configuration.Workflow{})
+	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, diggerConfig.Workflows)
 	assert.NoError(t, err)
 	_, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, &dynamoDbLock, planStorage, dir)
 	assert.NoError(t, err)
@@ -588,7 +588,7 @@ func TestMultiEnvHappyPath(t *testing.T) {
 	// 'digger apply' comment should trigger terraform execution and unlock the project
 	impactedProjects, prNumber, _, err = digger.ProcessGitHubEvent(ghEvent, diggerConfig, githubPrService)
 	assert.NoError(t, err)
-	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, map[string]configuration.Workflow{})
+	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, diggerConfig.Workflows)
 	assert.NoError(t, err)
 	_, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, &dynamoDbLock, planStorage, dir)
 	assert.NoError(t, err)
@@ -612,7 +612,7 @@ func TestMultiEnvHappyPath(t *testing.T) {
 
 	impactedProjects, prNumber, _, err = digger.ProcessGitHubEvent(ghEvent, diggerConfig, githubPrService)
 	assert.NoError(t, err)
-	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, map[string]configuration.Workflow{})
+	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, diggerConfig.Workflows)
 	assert.NoError(t, err)
 	_, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, &dynamoDbLock, planStorage, dir)
 	assert.NoError(t, err)
@@ -754,7 +754,7 @@ workflows:
 	//  new pr should lock the project
 	impactedProjects, prNumber, _, err := digger.ProcessGitHubEvent(ghEvent, diggerConfig, githubPrService)
 	assert.NoError(t, err)
-	commandsToRunPerProject, err := digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, map[string]configuration.Workflow{})
+	commandsToRunPerProject, err := digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, diggerConfig.Workflows)
 	assert.NoError(t, err)
 
 	zipManager := utils.Zipper{}
@@ -790,7 +790,7 @@ workflows:
 	// 'digger plan' comment should trigger terraform execution
 	impactedProjects, prNumber, _, err = digger.ProcessGitHubEvent(ghEvent, diggerConfig, githubPrService)
 	assert.NoError(t, err)
-	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, map[string]configuration.Workflow{})
+	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, diggerConfig.Workflows)
 	assert.NoError(t, err)
 	_, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, lock, planStorage, dir)
 	assert.NoError(t, err)
@@ -804,7 +804,7 @@ workflows:
 	// 'digger apply' comment should trigger terraform execution and unlock the project
 	impactedProjects, prNumber, _, err = digger.ProcessGitHubEvent(ghEvent, diggerConfig, githubPrService)
 	assert.NoError(t, err)
-	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, map[string]configuration.Workflow{})
+	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, diggerConfig.Workflows)
 	assert.NoError(t, err)
 	_, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, lock, planStorage, dir)
 	assert.NoError(t, err)
@@ -827,7 +827,7 @@ workflows:
 
 	impactedProjects, prNumber, _, err = digger.ProcessGitHubEvent(ghEvent, diggerConfig, githubPrService)
 	assert.NoError(t, err)
-	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, map[string]configuration.Workflow{})
+	commandsToRunPerProject, err = digger.ConvertGithubEventToCommands(ghEvent, impactedProjects, diggerConfig.Workflows)
 	assert.NoError(t, err)
 	_, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, lock, planStorage, dir)
 	assert.NoError(t, err)
diff --git a/pkg/terraform/tf.go b/pkg/terraform/tf.go
diff --git a/pkg/terraform/tf_test.go b/pkg/terraform/tf_test.go

Original file line number	Diff line number	Diff line change
`@@ -30,20 +30,20 @@ type MockTerraformExecutor struct {`
`30`	`30`	`Commands []RunInfo`
`31`	`31`	`}`
`32`	`32`
`33`		`-func (m *MockTerraformExecutor) Init(params []string) (string, string, error) {`
	`33`	`+func (m *MockTerraformExecutor) Init(params []string, envs map[string]string) (string, string, error) {`
`34`	`34`	`m.Commands = append(m.Commands, RunInfo{"Init", strings.Join(params, " "), time.Now()})`
`35`	`35`	`return "", "", nil`
`36`	`36`	`}`
`37`	`37`
`38`		`-func (m MockTerraformExecutor) Apply(params []string, plan string) (string, string, error) {`
	`38`	`+func (m MockTerraformExecutor) Apply(params []string, plan string, envs map[string]string) (string, string, error) {`
`39`	`39`	`if plan != nil {`
`40`	`40`	`params = append(params, *plan)`
`41`	`41`	`}`
`42`	`42`	`m.Commands = append(m.Commands, RunInfo{"Apply", strings.Join(params, " "), time.Now()})`
`43`	`43`	`return "", "", nil`
`44`	`44`	`}`
`45`	`45`
`46`		`-func (m *MockTerraformExecutor) Plan(params []string) (bool, string, string, error) {`
	`46`	`+func (m *MockTerraformExecutor) Plan(params []string, envs map[string]string) (bool, string, string, error) {`
`47`	`47`	`m.Commands = append(m.Commands, RunInfo{"Plan", strings.Join(params, " "), time.Now()})`
`48`	`48`	`return true, "", "", nil`
`49`	`49`	`}`