Merge pull request #69 from buildkite/add-secrets-support

Support pipeline `secrets` syntax

Author: matthewborden

pipeline.go

@@ -13,8 +13,9 @@ import (
 //
 // Standard caveats apply - see the package comment.
 type Pipeline struct {
-	Steps Steps          `yaml:"steps"`
-	Env   *ordered.MapSS `yaml:"env,omitempty"`
+	Steps   Steps          `yaml:"steps"`
+	Env     *ordered.MapSS `yaml:"env,omitempty"`
+	Secrets Secrets        `yaml:"secrets,omitempty"`
 
 	// RemainingFields stores any other top-level mapping items so they at least
 	// survive an unmarshal-marshal round-trip.

pipeline_secrets_test.go

@@ -0,0 +1,118 @@
+package pipeline
+
+import (
+	"testing"
+
+	"github.com/buildkite/go-pipeline/ordered"
+	"github.com/google/go-cmp/cmp"
+	"gopkg.in/yaml.v3"
+)
+
+func TestPipelineSecretsUnmarshal(t *testing.T) {
+	t.Parallel()
+
+	// Test parsing pipeline with build-level secrets
+	yamlData := `
+secrets:
+  - DATABASE_URL
+  - API_TOKEN
+steps:
+  - command: echo "hello"
+`
+
+	var p Pipeline
+	var node yaml.Node
+	err := yaml.Unmarshal([]byte(yamlData), &node)
+	if err != nil {
+		t.Fatalf("yaml.Unmarshal() error = %v", err)
+	}
+
+	err = ordered.Unmarshal(&node, &p)
+	if err != nil {
+		t.Fatalf("ordered.Unmarshal() error = %v", err)
+	}
+
+	want := Secrets{
+		{Key: "DATABASE_URL", EnvironmentVariable: "DATABASE_URL"},
+		{Key: "API_TOKEN", EnvironmentVariable: "API_TOKEN"},
+	}
+
+	if diff := cmp.Diff(p.Secrets, want); diff != "" {
+		t.Errorf("p.Secrets = %v, want %v", p.Secrets, want)
+	}
+}
+
+func TestPipelineSecretsWithSteps(t *testing.T) {
+	t.Parallel()
+
+	// Test complete pipeline with both pipeline and step secrets
+	yamlData := `
+secrets:
+  - DATABASE_URL
+  - REDIS_URL
+steps:
+  - command: echo "step1"
+    secrets:
+      - API_TOKEN
+      - DATABASE_URL
+  - command: echo "step2"
+`
+
+	var p Pipeline
+	var node yaml.Node
+	err := yaml.Unmarshal([]byte(yamlData), &node)
+	if err != nil {
+		t.Fatalf("yaml.Unmarshal() error = %v", err)
+	}
+
+	err = ordered.Unmarshal(&node, &p)
+	if err != nil {
+		t.Fatalf("ordered.Unmarshal() error = %v", err)
+	}
+
+	want := Secrets{
+		{Key: "DATABASE_URL", EnvironmentVariable: "DATABASE_URL"},
+		{Key: "REDIS_URL", EnvironmentVariable: "REDIS_URL"},
+	}
+
+	if diff := cmp.Diff(want, p.Secrets); diff != "" {
+		t.Errorf("unexpected secrets (-want +got):\n%s", diff)
+	}
+
+	// Check steps
+	if len(p.Steps) != 2 {
+		t.Fatalf("len(p.Steps) = %d, want 2", len(p.Steps))
+	}
+
+	// First step should have its own secrets
+	step1, ok := p.Steps[0].(*CommandStep)
+	if !ok {
+		t.Fatalf("p.Steps[0] is not a CommandStep")
+	}
+	if len(step1.Secrets) != 2 {
+		t.Fatalf("len(step1.Secrets) = %d, want 2", len(step1.Secrets))
+	}
+
+	// Second step should have no secrets initially
+	step2, ok := p.Steps[1].(*CommandStep)
+	if !ok {
+		t.Fatalf("p.Steps[1] is not a CommandStep")
+	}
+	if len(step2.Secrets) != 0 {
+		t.Fatalf("len(step2.Secrets) = %d, want 0", len(step2.Secrets))
+	}
+
+	// Test merging for both steps
+	step1.MergeSecretsFromPipeline(p.Secrets)
+	step2.MergeSecretsFromPipeline(p.Secrets)
+
+	// Step1 should have 3 secrets (2 from step + 1 from pipeline not overridden)
+	if len(step1.Secrets) != 3 {
+		t.Fatalf("len(step1.Secrets after merge) = %d, want 3", len(step1.Secrets))
+	}
+
+	// Step2 should have 2 secrets (both from pipeline)
+	if len(step2.Secrets) != 2 {
+		t.Fatalf("len(step2.Secrets after merge) = %d, want 2", len(step2.Secrets))
+	}
+}

secret.go

@@ -0,0 +1,43 @@
+package pipeline
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+var (
+	_ interface {
+		json.Marshaler
+		selfInterpolater
+	} = (*Secret)(nil)
+)
+
+// Secret represents a pipeline secret configuration.
+type Secret struct {
+	Key                 string         `json:"key" yaml:"key"`
+	EnvironmentVariable string         `json:"environment_variable,omitempty" yaml:"environment_variable,omitempty"`
+	RemainingFields     map[string]any `yaml:",inline"`
+}
+
+// MarshalJSON marshals the secret to JSON.
+func (s Secret) MarshalJSON() ([]byte, error) {
+	return inlineFriendlyMarshalJSON(s)
+}
+
+func (s *Secret) interpolate(tf stringTransformer) error {
+	key, err := tf.Transform(s.Key)
+	if err != nil {
+		return fmt.Errorf("interpolating secret key: %w", err)
+	}
+	s.Key = key
+
+	if s.EnvironmentVariable != "" {
+		envVar, err := tf.Transform(s.EnvironmentVariable)
+		if err != nil {
+			return fmt.Errorf("interpolating environment variable: %w", err)
+		}
+		s.EnvironmentVariable = envVar
+	}
+
+	return nil
+}

secret_test.go

@@ -0,0 +1,78 @@
+package pipeline
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/buildkite/interpolate"
+	"github.com/google/go-cmp/cmp"
+	"gopkg.in/yaml.v3"
+)
+
+func TestSecretMarshalJSON(t *testing.T) {
+	t.Parallel()
+
+	secret := Secret{
+		Key:                 "DATABASE_URL",
+		EnvironmentVariable: "DATABASE_URL",
+	}
+
+	want := `{"environment_variable":"DATABASE_URL","key":"DATABASE_URL"}`
+	got, err := json.Marshal(secret)
+	if err != nil {
+		t.Fatalf("json.Marshal(%#v) error = %v", secret, err)
+	}
+
+	if string(got) != want {
+		t.Errorf("json.Marshal(%#v) = %q, want %q", secret, got, want)
+	}
+}
+
+func TestSecretMarshalYAML(t *testing.T) {
+	t.Parallel()
+
+	secret := Secret{
+		Key:                 "DATABASE_URL",
+		EnvironmentVariable: "DATABASE_URL",
+	}
+
+	got, err := yaml.Marshal(secret)
+	if err != nil {
+		t.Fatalf("yaml.Marshal(%#v) error = %v", secret, err)
+	}
+
+	want := "key: DATABASE_URL\nenvironment_variable: DATABASE_URL\n"
+	if string(got) != want {
+		t.Errorf("yaml.Marshal(%#v) = %q, want %q", secret, got, want)
+	}
+}
+
+func TestSecretInterpolation(t *testing.T) {
+	t.Parallel()
+
+	secret := Secret{
+		Key:                 "${SECRET_NAME}",
+		EnvironmentVariable: "${ENV_VAR_NAME}",
+	}
+
+	tf := envInterpolator{
+		env: interpolate.NewMapEnv(map[string]string{
+			"SECRET_NAME":  "DATABASE_URL",
+			"ENV_VAR_NAME": "DB_CONNECTION",
+		}),
+	}
+
+	err := secret.interpolate(tf)
+	if err != nil {
+		t.Fatalf("secret.interpolate(%#v) error = %v", tf, err)
+	}
+
+	want := Secret{
+		Key:                 "DATABASE_URL",
+		EnvironmentVariable: "DB_CONNECTION",
+	}
+
+	if diff := cmp.Diff(want, secret); diff != "" {
+		t.Errorf("secret.interpolate(%#v) = %q, want: %q", tf, diff, want)
+	}
+}

secrets.go

@@ -0,0 +1,141 @@
+package pipeline
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/buildkite/go-pipeline/ordered"
+	"gopkg.in/yaml.v3"
+)
+
+var _ interface {
+	json.Unmarshaler
+	ordered.Unmarshaler
+	yaml.Marshaler
+} = (*Secrets)(nil)
+
+// Secrets is a sequence of secrets. It is useful for unmarshaling.
+type Secrets []Secret
+
+// UnmarshalOrdered unmarshals Secrets from []any (sequence of secret names).
+func (s *Secrets) UnmarshalOrdered(o any) error {
+	switch o := o.(type) {
+	case nil:
+		// `secrets: null` is invalid - should be omitted entirely or use valid formats
+		return fmt.Errorf("unmarshaling secrets: secrets cannot be null")
+
+	case *ordered.Map[string, any]:
+		// Handle map syntax: {"ENV_VAR": "SECRET_KEY"}
+		return o.Range(func(envVar string, secretKeyVal any) error {
+			secretKey, ok := secretKeyVal.(string)
+			if !ok {
+				return fmt.Errorf("unmarshaling secrets: secret key must be a string, but was %T", secretKeyVal)
+			}
+			if secretKey == "" {
+				return fmt.Errorf("unmarshaling secrets: secret key cannot be empty")
+			}
+			if envVar == "" {
+				return fmt.Errorf("unmarshaling secrets: environment variable name cannot be empty")
+			}
+
+			secret := Secret{
+				Key:                 secretKey,
+				EnvironmentVariable: envVar,
+			}
+			*s = append(*s, secret)
+			return nil
+		})
+
+	case []any:
+		for _, c := range o {
+			switch ct := c.(type) {
+			case string:
+				secret := Secret{
+					Key:                 ct,
+					EnvironmentVariable: ct, // Default EnvironmentVariable to key value for simple string format
+				}
+				*s = append(*s, secret)
+
+			case *ordered.Map[string, interface{}]:
+				// Backend sends ordered.Map format
+				secret := Secret{}
+
+				keyVal, _ := ct.Get("key")
+				key, _ := keyVal.(string)
+				if key == "" {
+					return fmt.Errorf("unmarshaling secret: key must be a non-empty string, but was %[1]T %[1]v", keyVal)
+				}
+				secret.Key = key
+
+				if envVarVal, _ := ct.Get("environment_variable"); envVarVal != nil {
+					envVar, ok := envVarVal.(string)
+					if !ok {
+						return fmt.Errorf("unmarshaling secret: environment_variable must be a string, but was %T", envVarVal)
+					}
+					secret.EnvironmentVariable = envVar
+				}
+
+				*s = append(*s, secret)
+
+			default:
+				return fmt.Errorf("unmarshaling secrets: secret type %T, want string, map[string]any, or *ordered.Map", c)
+			}
+		}
+
+	default:
+		return fmt.Errorf("unmarshaling secrets: got %T, want []any or map[string]any", o)
+	}
+
+	return nil
+}
+
+// MergeWith merges these secrets with another set of secrets, with the other secrets taking precedence.
+// Deduplication is performed based on the EnvironmentVariable field.
+func (s Secrets) MergeWith(other Secrets) Secrets {
+	if len(s) == 0 {
+		return other
+	}
+	if len(other) == 0 {
+		return s
+	}
+
+	// Create a map to track environment variables we've seen for deduplication
+	seen := make(map[string]bool)
+	var result Secrets
+
+	for _, secret := range other {
+		if secret.EnvironmentVariable != "" && !seen[secret.EnvironmentVariable] {
+			result = append(result, secret)
+			seen[secret.EnvironmentVariable] = true
+		}
+	}
+
+	for _, secret := range s {
+		if secret.EnvironmentVariable != "" && !seen[secret.EnvironmentVariable] {
+			result = append(result, secret)
+			seen[secret.EnvironmentVariable] = true
+		}
+	}
+
+	return result
+}
+
+// UnmarshalJSON is used for JSON unmarshaling.
+func (s *Secrets) UnmarshalJSON(b []byte) error {
+	// JSON is just a specific kind of YAML.
+	var n yaml.Node
+	if err := yaml.Unmarshal(b, &n); err != nil {
+		return err
+	}
+	return ordered.Unmarshal(&n, &s)
+}
+
+func (s Secrets) MarshalYAML() (any, error) {
+	if len(s) == 0 {
+		return nil, nil
+	}
+
+	result := make([]Secret, len(s))
+	copy(result, s)
+	return result, nil
+}