Add validate_dependencies flag to pipeline upload

Add test for pipeline upload params

Author: dabarrell

agent/pipeline_uploader_test.go

@@ -2,6 +2,7 @@ package agent_test
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -17,6 +18,66 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func TestPipelineUploadParameters(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	l := logger.NewBuffer()
+
+	jobID := api.NewUUID()
+	stepUploadUUID := api.NewUUID()
+	pipelineStr := `---
+steps:
+  - command: "echo Hello"`
+
+	pipeline, err := pipeline.Parse(strings.NewReader(pipelineStr))
+	assert.NoError(t, err)
+
+	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		switch req.URL.Path {
+		case fmt.Sprintf("/jobs/%s/pipelines", jobID):
+			var pipelineUpload map[string]any
+			if err := json.NewDecoder(req.Body).Decode(&pipelineUpload); err != nil {
+				http.Error(rw, "Error decoding pipeline", http.StatusInternalServerError)
+				return
+			}
+			assert.Equal(t, pipelineUpload["validate_dependencies"], true)
+			assert.Equal(t, pipelineUpload["replace"], true)
+
+			if req.Method == "POST" {
+				rw.Header().Add("Location", fmt.Sprintf("/jobs/%s/pipelines/%s", jobID, stepUploadUUID))
+				rw.WriteHeader(http.StatusAccepted)
+				return
+			}
+		case fmt.Sprintf("/jobs/%s/pipelines/%s", jobID, stepUploadUUID):
+			if req.Method == "GET" {
+				_, _ = fmt.Fprintf(rw, `{"state": "%s"}`, "applied")
+				return
+			}
+		}
+		t.Errorf("Unknown endpoint %s %s", req.Method, req.URL.Path)
+		http.Error(rw, "Not found", http.StatusNotFound)
+	}))
+	defer server.Close()
+
+	uploader := &agent.PipelineUploader{
+		Client: api.NewClient(logger.Discard, api.Config{
+			Endpoint: server.URL,
+			Token:    "llamas",
+		}),
+		JobID: jobID,
+		Change: &api.PipelineChange{
+			UUID:                 stepUploadUUID,
+			Pipeline:             pipeline,
+			Replace:              true,
+			ValidateDependencies: true,
+		},
+	}
+
+	err = uploader.Upload(ctx, l)
+	assert.NoError(t, err)
+}
+
 func TestAsyncPipelineUpload(t *testing.T) {
 	t.Parallel()
 

api/pipelines.go

@@ -9,9 +9,10 @@ import (
 type PipelineChange struct {
 	// UUID identifies this pipeline change. We keep this constant during
 	// retry loops so that work is not repeated on the API server
-	UUID     string `json:"uuid"`
-	Pipeline any    `json:"pipeline"`
-	Replace  bool   `json:"replace,omitempty"`
+	UUID                 string `json:"uuid"`
+	Pipeline             any    `json:"pipeline"`
+	Replace              bool   `json:"replace,omitempty"`
+	ValidateDependencies bool   `json:"validate_dependencies,omitempty"`
 }
 
 type PipelineUploadStatus struct {

clicommand/pipeline_upload.go

@@ -74,14 +74,15 @@ type PipelineUploadConfig struct {
 	GlobalConfig
 	APIConfig
 
-	FilePath        string   `cli:"arg:0" label:"upload paths"`
-	Replace         bool     `cli:"replace"`
-	Job             string   `cli:"job"` // required, but not in dry-run mode
-	DryRun          bool     `cli:"dry-run"`
-	DryRunFormat    string   `cli:"format"`
-	NoInterpolation bool     `cli:"no-interpolation"`
-	RedactedVars    []string `cli:"redacted-vars" normalize:"list"`
-	RejectSecrets   bool     `cli:"reject-secrets"`
+	FilePath             string   `cli:"arg:0" label:"upload paths"`
+	Replace              bool     `cli:"replace"`
+	Job                  string   `cli:"job"` // required, but not in dry-run mode
+	DryRun               bool     `cli:"dry-run"`
+	DryRunFormat         string   `cli:"format"`
+	NoInterpolation      bool     `cli:"no-interpolation"`
+	RedactedVars         []string `cli:"redacted-vars" normalize:"list"`
+	RejectSecrets        bool     `cli:"reject-secrets"`
+	ValidateDependencies bool     `cli:"validate-dependencies"`
 
 	// Used for if_changed processing
 	ApplyIfChanged bool   `cli:"apply-if-changed"`
@@ -142,7 +143,11 @@ var PipelineUploadCommand = cli.Command{
 			Value:  "origin/main",
 			EnvVar: "BUILDKITE_GIT_DIFF_BASE,BUILDKITE_PULL_REQUEST_BASE_BRANCH",
 		},
-
+		cli.BoolFlag{
+			Name:   "validate-dependencies",
+			Usage:  "When true, validates the dependencies of each step when uploading.",
+			EnvVar: "BUILDKITE_PIPELINE_VALIDATE_DEPENDENCIES",
+		},
 		// Note: changes to these environment variables need to be reflected in the environment created
 		// in the job runner. At the momenet, that's at agent/job_runner.go:500-507
 		cli.StringFlag{
@@ -381,9 +386,10 @@ var PipelineUploadCommand = cli.Command{
 				Client: api.NewClient(l, loadAPIClientConfig(cfg, "AgentAccessToken")),
 				JobID:  cfg.Job,
 				Change: &api.PipelineChange{
-					UUID:     api.NewUUID(),
-					Replace:  cfg.Replace,
-					Pipeline: result,
+					UUID:                 api.NewUUID(),
+					Replace:              cfg.Replace,
+					Pipeline:             result,
+					ValidateDependencies: cfg.ValidateDependencies,
 				},
 				RetrySleepFunc: time.Sleep,
 			}

clicommand/pipeline_upload_test.go

@@ -18,8 +18,9 @@ func TestSearchForSecrets(t *testing.T) {
 	t.Parallel()
 
 	cfg := &PipelineUploadConfig{
-		RedactedVars:  []string{"SEKRET", "SSH_KEY"},
-		RejectSecrets: true,
+		RedactedVars:         []string{"SEKRET", "SSH_KEY"},
+		RejectSecrets:        true,
+		ValidateDependencies: false,
 	}
 
 	plainPipeline := &pipeline.Pipeline{