[CLD-870]: fix: allow empty payload field in input file (#607)

Previously we reject any input with empty payload field in the pipeline
input, however there are scenarios where user wants to pass in empty
input
[here](https://chainlink-core.slack.com/archives/C08QF1BEW4T/p1764199008922179)

```
environment: testnet 
domain: ccip
changesets:
  propose_jobs: # Changeset 1
    chainOverrides:
      - 16015286601757825753   # list, not bare map key
      - 9762610643973837292
    payload:  # empty 
```

Solution: Changed the validation to check for the existence of the key
rather than the truthiness of the value

Now when we pass {"deploy_link_token": {"payload": null}}, the
validation will correctly recognize that the payload field exists (even
though its value is null), and the pipeline will proceed successfully.
This allows you to explicitly pass nil payloads when needed.

JIRA: https://smartcontract-it.atlassian.net/browse/CLD-870

Author: graham-chainlink

.changeset/silver-chairs-wish.md

@@ -0,0 +1,5 @@
+---
+"chainlink-deployments-framework": patch
+---
+
+fix: allow empty payload field in input file

engine/cld/changeset/common.go

@@ -100,9 +100,10 @@ func (f WrappedChangeSet[C]) WithJSON(_ C, inputStr string) ConfiguredChangeSet
 			return config, fmt.Errorf("JSON must be in JSON format with 'payload' fields: %w", err)
 		}
 
-		// If payload is null or empty, return an error
-		if len(inputObject.Payload) == 0 || string(inputObject.Payload) == "null" {
-			return config, errors.New("'payload' field is required and cannot be empty")
+		// If payload is null, decode it as null (which will give zero value)
+		// If payload is missing, return an error
+		if len(inputObject.Payload) == 0 {
+			return config, errors.New("'payload' field is required")
 		}
 
 		payloadDecoder := json.NewDecoder(strings.NewReader(string(inputObject.Payload)))
@@ -159,9 +160,10 @@ func (f WrappedChangeSet[C]) WithEnvInput(opts ...EnvInputOption[C]) ConfiguredC
 			return config, fmt.Errorf("JSON must be in JSON format with 'payload' fields: %w", err)
 		}
 
-		// If payload is null or empty, return an error
-		if len(inputObject.Payload) == 0 || string(inputObject.Payload) == "null" {
-			return config, errors.New("'payload' field is required and cannot be empty")
+		// If payload is null, decode it as null (which will give zero value)
+		// If payload is missing, return an error
+		if len(inputObject.Payload) == 0 {
+			return config, errors.New("'payload' field is required")
 		}
 
 		payloadDecoder := json.NewDecoder(strings.NewReader(string(inputObject.Payload)))
@@ -229,9 +231,10 @@ func (f WrappedChangeSet[C]) WithConfigResolver(resolver fresolvers.ConfigResolv
 			return zero, fmt.Errorf("failed to parse resolver input as JSON: %w", err)
 		}
 
-		// If payload is null or empty, return an error
-		if len(inputObject.Payload) == 0 || string(inputObject.Payload) == "null" {
-			return zero, errors.New("'payload' field is required and cannot be empty")
+		// If payload is null, pass it to the resolver (which will receive null)
+		// If payload field is missing, return an error
+		if len(inputObject.Payload) == 0 {
+			return zero, errors.New("'payload' field is required")
 		}
 
 		// Call resolver – automatically unmarshal into its expected input type.

engine/cld/changeset/common_test.go

@@ -449,14 +449,135 @@ func TestWithConfigResolver_InvalidJSON(t *testing.T) {
 	assert.Contains(t, err.Error(), "failed to parse resolver input as JSON")
 }
 
+func TestWithConfigResolver_NullPayload(t *testing.T) {
+	manager := resolvers.NewConfigResolverManager()
+	resolver := func(input map[string]any) (any, error) {
+		// When payload is null, input will be nil
+		if input == nil {
+			return "resolved_from_null", nil
+		}
+
+		return "should_not_reach_here", nil
+	}
+	manager.Register(resolver, resolvers.ResolverInfo{Description: "Test"})
+
+	t.Setenv("DURABLE_PIPELINE_INPUT", `{"payload": null}`)
+
+	cs := deployment.CreateChangeSet(
+		func(e deployment.Environment, config string) (deployment.ChangesetOutput, error) {
+			assert.Equal(t, "resolved_from_null", config)
+			return deployment.ChangesetOutput{}, nil
+		},
+		func(e deployment.Environment, config string) error { return nil },
+	)
+
+	configured := Configure(cs).WithConfigResolver(resolver)
+	env := deployment.Environment{Logger: logger.Test(t)}
+
+	_, err := configured.Apply(env)
+	require.NoError(t, err, "null payload should be valid and passed to resolver")
+}
+
+func TestWithJSON_NullPayload(t *testing.T) {
+	t.Parallel()
+
+	type TestConfig struct {
+		Value string `json:"value"`
+		Count int    `json:"count"`
+	}
+
+	cs := deployment.CreateChangeSet(
+		func(e deployment.Environment, config TestConfig) (deployment.ChangesetOutput, error) {
+			// When payload is null, config should be zero value
+			assert.Empty(t, config.Value)
+			assert.Equal(t, 0, config.Count)
+
+			return deployment.ChangesetOutput{}, nil
+		},
+		func(e deployment.Environment, config TestConfig) error { return nil },
+	)
+	env := deployment.Environment{Logger: logger.Test(t)}
+	configured := Configure(cs).WithJSON(TestConfig{}, `{"payload":null}`)
+
+	_, err := configured.Apply(env)
+	require.NoError(t, err, "null payload should be valid")
+}
+
+func TestWithJSON_MissingPayload(t *testing.T) {
+	t.Parallel()
+
+	type TestConfig struct {
+		Value string `json:"value"`
+	}
+
+	cs := deployment.CreateChangeSet(
+		func(e deployment.Environment, config TestConfig) (deployment.ChangesetOutput, error) {
+			return deployment.ChangesetOutput{}, nil
+		},
+		func(e deployment.Environment, config TestConfig) error { return nil },
+	)
+	env := deployment.Environment{Logger: logger.Test(t)}
+	configured := Configure(cs).WithJSON(TestConfig{}, `{"notPayload":"value"}`)
+
+	_, err := configured.Apply(env)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "'payload' field is required")
+}
+
+func TestWithEnvInput_NullPayload(t *testing.T) {
+	type TestConfig struct {
+		Value string `json:"value"`
+		Count int    `json:"count"`
+	}
+
+	t.Setenv("DURABLE_PIPELINE_INPUT", `{"payload":null}`)
+
+	cs := deployment.CreateChangeSet(
+		func(e deployment.Environment, config TestConfig) (deployment.ChangesetOutput, error) {
+			// When payload is null, config should be zero value
+			assert.Empty(t, config.Value)
+			assert.Equal(t, 0, config.Count)
+
+			return deployment.ChangesetOutput{}, nil
+		},
+		func(e deployment.Environment, config TestConfig) error { return nil },
+	)
+	env := deployment.Environment{Logger: logger.Test(t)}
+	configured := Configure(cs).WithEnvInput()
+
+	_, err := configured.Apply(env)
+	require.NoError(t, err, "null payload should be valid")
+}
+
+func TestWithEnvInput_MissingPayload(t *testing.T) {
+	type TestConfig struct {
+		Value string
+	}
+
+	t.Setenv("DURABLE_PIPELINE_INPUT", `{"notPayload":"value"}`)
+
+	cs := deployment.CreateChangeSet(
+		func(e deployment.Environment, config TestConfig) (deployment.ChangesetOutput, error) {
+			return deployment.ChangesetOutput{}, nil
+		},
+		func(e deployment.Environment, config TestConfig) error { return nil },
+	)
+	env := deployment.Environment{Logger: logger.Test(t)}
+	configured := Configure(cs).WithEnvInput()
+
+	_, err := configured.Apply(env)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "'payload' field is required")
+}
+
 func TestWithConfigResolver_MissingPayload(t *testing.T) {
 	manager := resolvers.NewConfigResolverManager()
 	resolver := func(input map[string]any) (any, error) {
 		return "config", nil
 	}
 	manager.Register(resolver, resolvers.ResolverInfo{Description: "Test"})
 
-	t.Setenv("DURABLE_PIPELINE_INPUT", `{"payload": null}`)
+	t.Setenv("DURABLE_PIPELINE_INPUT", `{"notPayload":"value"}`)
 
 	cs := deployment.CreateChangeSet(
 		func(e deployment.Environment, config string) (deployment.ChangesetOutput, error) {
@@ -470,7 +591,7 @@ func TestWithConfigResolver_MissingPayload(t *testing.T) {
 
 	_, err := configured.Apply(env)
 	require.Error(t, err)
-	assert.Contains(t, err.Error(), "'payload' field is required and cannot be empty")
+	assert.Contains(t, err.Error(), "'payload' field is required")
 }
 
 func TestWithConfigResolver_ResolverError(t *testing.T) {

engine/cld/legacy/cli/commands/durable-pipelines_test.go

@@ -1506,6 +1506,17 @@ changesets:
 			expectError:   true,
 			errorContains: "is missing required 'payload' field",
 		},
+		{
+			name: "null payload field - should be valid",
+			yamlContent: `environment: testnet
+domain: test
+changesets:
+  - test_changeset:
+      payload: null`,
+			index:        0,
+			expectedName: "test_changeset",
+			expectedJSON: `{"payload":null}`,
+		},
 	}
 
 	for _, tt := range tests {
@@ -1740,6 +1751,114 @@ changesets: [
 	}
 }
 
+//nolint:paralleltest
+func TestSetDurablePipelineInputFromYAML_NullPayload(t *testing.T) {
+	testDomain := domain.NewDomain(t.TempDir(), "test")
+	env := "testnet"
+
+	// Create workspace structure
+	workspaceRoot := t.TempDir()
+	inputsDir := filepath.Join(workspaceRoot, "domains", testDomain.String(), env, "durable_pipelines", "inputs")
+	require.NoError(t, os.MkdirAll(inputsDir, 0755))
+
+	// Mock workspace root discovery
+	require.NoError(t, os.MkdirAll(filepath.Join(workspaceRoot, "domains"), 0755))
+
+	// Set up the test to run from within the workspace
+	originalWd, err := os.Getwd()
+	require.NoError(t, err)
+	require.NoError(t, os.Chdir(workspaceRoot))
+	t.Cleanup(func() {
+		require.NoError(t, os.Chdir(originalWd))
+	})
+
+	tests := []struct {
+		name          string
+		yamlContent   string
+		changesetName string
+		expectError   bool
+		expectedJSON  string
+		description   string
+	}{
+		{
+			name: "object format with null payload - should be valid",
+			yamlContent: `environment: testnet
+domain: test
+changesets:
+  deploy_link_token:
+    payload: null`,
+			changesetName: "deploy_link_token",
+			expectError:   false,
+			expectedJSON:  `{"payload":null}`,
+			description:   "Should allow explicit null payload in object format",
+		},
+		{
+			name: "array format with null payload - should be valid",
+			yamlContent: `environment: testnet
+domain: test
+changesets:
+  - deploy_link_token:
+      payload: null`,
+			changesetName: "deploy_link_token",
+			expectError:   false,
+			expectedJSON:  `{"payload":null}`,
+			description:   "Should allow explicit null payload in array format",
+		},
+		{
+			name: "object format with missing payload - should error",
+			yamlContent: `environment: testnet
+domain: test
+changesets:
+  deploy_link_token:
+    notPayload: 123`,
+			changesetName: "deploy_link_token",
+			expectError:   true,
+			description:   "Should error when payload field is completely missing",
+		},
+		{
+			name: "object format with empty payload object - should be valid",
+			yamlContent: `environment: testnet
+domain: test
+changesets:
+  deploy_link_token:
+    payload: {}`,
+			changesetName: "deploy_link_token",
+			expectError:   false,
+			expectedJSON:  `{"payload":{}}`,
+			description:   "Should allow empty object as payload",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Create unique YAML file for this test
+			safeName := strings.ReplaceAll(strings.ReplaceAll(tt.name, " ", "-"), "/", "-")
+			yamlFileName := fmt.Sprintf("test-null-payload-%s.yaml", safeName)
+			yamlFilePath := filepath.Join(inputsDir, yamlFileName)
+			require.NoError(t, os.WriteFile(yamlFilePath, []byte(tt.yamlContent), 0644)) //nolint:gosec
+
+			// Clear any previous DURABLE_PIPELINE_INPUT
+			os.Unsetenv("DURABLE_PIPELINE_INPUT")
+
+			err := setDurablePipelineInputFromYAML(yamlFileName, tt.changesetName, testDomain, env)
+
+			if tt.expectError {
+				require.Error(t, err, tt.description)
+				require.Contains(t, err.Error(), "is missing required 'payload' field", tt.description)
+			} else {
+				require.NoError(t, err, tt.description)
+
+				// Verify that DURABLE_PIPELINE_INPUT was set
+				durablePipelineInput := os.Getenv("DURABLE_PIPELINE_INPUT")
+				require.NotEmpty(t, durablePipelineInput, "DURABLE_PIPELINE_INPUT should be set")
+
+				// Verify the JSON structure
+				require.JSONEq(t, tt.expectedJSON, durablePipelineInput, tt.description)
+			}
+		})
+	}
+}
+
 //nolint:paralleltest
 func TestDurablePipelineRunWithObjectFormatError(t *testing.T) {
 	env := "testnet"

engine/cld/legacy/cli/commands/durable_pipeline_yaml.go

@@ -205,7 +205,7 @@ func setChangesetEnvironmentVariable(changesetName string, changesetData any, in
 	}
 
 	payload, payloadExists := changesetMap["payload"]
-	if !payloadExists || payload == nil {
+	if !payloadExists {
 		return fmt.Errorf("changeset '%s' in input file %s is missing required 'payload' field", changesetName, inputFileName)
 	}