feat(status): add pending_as_failure option to report failure instead of pending

When a PR doesn't meet approval policy requirements but hasn't been
explicitly disapproved, policy-bot normally reports a "pending" GitHub
status. This breaks tools like Kodiak that rely on definitive pass/fail
status checks and remain stuck waiting for the status to resolve.

This change adds a new `pending_as_failure` option that reports "failure"
instead of "pending" for such PRs.

Configuration options:

1. Server-level (policy-bot.yml):
   ```yaml
   options:
     pending_as_failure: true
   ```

2. Environment variable:
   ```
   POLICYBOT_OPTIONS_PENDING_AS_FAILURE=true
   ```

3. Per-repo policy override (.policy.yml):
   ```yaml
   pending_as_failure: true  # or false to override server setting
   ```

The policy-level setting takes precedence over the server-level setting,
allowing repos to opt-in or opt-out individually.

Changes:
- server/handler/eval_options.go: Add PendingAsFailure field and env var
- policy/policy.go: Add PendingAsFailure to Config struct for per-repo override
- server/handler/eval_context.go: Check both settings when mapping StatusPending
- config/policy-bot.example.yml: Document the new option
- server/handler/eval_options_test.go: Add env var test
- server/handler/eval_context_test.go: Add comprehensive tests for all scenarios

Co-Authored-By: Claude <noreply@anthropic.com>

Author: denysvitali-kaiko

config/policy-bot.example.yml

@@ -148,6 +148,14 @@ sessions:
 #   # environment variable.
 #   force_shared_policy: false
 #
+#   # If true, report "failure" instead of "pending" for PRs that don't
+#   # meet approval requirements. Useful for tools like Kodiak that require
+#   # definitive pass/fail status checks. This can be overridden per-repo
+#   # by setting pending_as_failure in the .policy.yml file.
+#   # Can also be set by the POLICYBOT_OPTIONS_PENDING_AS_FAILURE
+#   # environment variable.
+#   pending_as_failure: false
+#
 #   # Default options to use for all appoval rules processed by the server.
 #   # Policies can override these values with their own defaults or in individual
 #   # rules. This is equivalent to the "approval_defaults" option in a policy and

policy/policy.go

@@ -38,6 +38,11 @@ type Config struct {
 	Policy           Policy             `yaml:"policy,omitempty"`
 	ApprovalDefaults *approval.Defaults `yaml:"approval_defaults,omitempty"`
 	ApprovalRules    []*approval.Rule   `yaml:"approval_rules,omitempty"`
+
+	// PendingAsFailure reports "failure" instead of "pending" for PRs that
+	// don't meet approval requirements. When set, this overrides the
+	// server-level setting.
+	PendingAsFailure *bool `yaml:"pending_as_failure,omitempty"`
 }
 
 type Policy struct {

server/handler/eval_context.go

@@ -145,7 +145,16 @@ func (ec *EvalContext) EvaluatePolicy(ctx context.Context, evaluator common.Eval
 	case common.StatusDisapproved:
 		statusState = "failure"
 	case common.StatusPending:
-		statusState = "pending"
+		// Policy-level setting takes precedence over server-level
+		pendingAsFailure := ec.Options.PendingAsFailure
+		if ec.Config.Config != nil && ec.Config.Config.PendingAsFailure != nil {
+			pendingAsFailure = *ec.Config.Config.PendingAsFailure
+		}
+		if pendingAsFailure {
+			statusState = "failure"
+		} else {
+			statusState = "pending"
+		}
 	case common.StatusSkipped:
 		statusState = "error"
 		statusDescription = "All rules were skipped. At least one rule must match."

server/handler/eval_context_test.go

@@ -0,0 +1,177 @@
+// Copyright 2025 Palantir Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handler
+
+import (
+	"context"
+	"testing"
+
+	"github.com/palantir/policy-bot/policy"
+	"github.com/palantir/policy-bot/policy/common"
+	"github.com/palantir/policy-bot/pull"
+	"github.com/palantir/policy-bot/pull/pulltest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type staticEvaluator common.Result
+
+func (eval *staticEvaluator) Trigger() common.Trigger {
+	return common.TriggerStatic
+}
+
+func (eval *staticEvaluator) Evaluate(ctx context.Context, prctx pull.Context) common.Result {
+	return common.Result(*eval)
+}
+
+func TestEvalContext_EvaluatePolicy_PendingAsFailure(t *testing.T) {
+	ctx := context.Background()
+
+	tests := map[string]struct {
+		serverPendingAsFailure bool
+		policyPendingAsFailure *bool
+		resultStatus           common.EvaluationStatus
+		expectedState          string
+	}{
+		"default_pending_returns_pending": {
+			serverPendingAsFailure: false,
+			policyPendingAsFailure: nil,
+			resultStatus:           common.StatusPending,
+			expectedState:          "pending",
+		},
+		"server_option_enabled_returns_failure": {
+			serverPendingAsFailure: true,
+			policyPendingAsFailure: nil,
+			resultStatus:           common.StatusPending,
+			expectedState:          "failure",
+		},
+		"policy_option_enabled_returns_failure": {
+			serverPendingAsFailure: false,
+			policyPendingAsFailure: ptr(true),
+			resultStatus:           common.StatusPending,
+			expectedState:          "failure",
+		},
+		"policy_option_overrides_server_to_disable": {
+			serverPendingAsFailure: true,
+			policyPendingAsFailure: ptr(false),
+			resultStatus:           common.StatusPending,
+			expectedState:          "pending",
+		},
+		"approved_returns_success_regardless": {
+			serverPendingAsFailure: true,
+			policyPendingAsFailure: nil,
+			resultStatus:           common.StatusApproved,
+			expectedState:          "success",
+		},
+		"disapproved_returns_failure_regardless": {
+			serverPendingAsFailure: false,
+			policyPendingAsFailure: nil,
+			resultStatus:           common.StatusDisapproved,
+			expectedState:          "failure",
+		},
+		"skipped_returns_error_regardless": {
+			serverPendingAsFailure: true,
+			policyPendingAsFailure: nil,
+			resultStatus:           common.StatusSkipped,
+			expectedState:          "error",
+		},
+		"both_server_and_policy_enabled": {
+			serverPendingAsFailure: true,
+			policyPendingAsFailure: ptr(true),
+			resultStatus:           common.StatusPending,
+			expectedState:          "failure",
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			prctx := &pulltest.Context{
+				OwnerValue:     "testowner",
+				RepoValue:      "testrepo",
+				NumberValue:    1,
+				HeadSHAValue:   "abc123",
+				BranchBaseName: "main",
+			}
+
+			policyConfig := &policy.Config{}
+			policyConfig.PendingAsFailure = test.policyPendingAsFailure
+
+			ec := &EvalContext{
+				Options: &PullEvaluationOptions{
+					StatusCheckContext: "policy-bot",
+					PendingAsFailure:   test.serverPendingAsFailure,
+				},
+				PublicURL:   "http://localhost:8080",
+				PullContext: prctx,
+				Config: FetchedConfig{
+					Config: policyConfig,
+					Source: "test",
+					Path:   ".policy.yml",
+				},
+				SkipPostStatus: true,
+			}
+
+			evaluator := &staticEvaluator{
+				Status:            test.resultStatus,
+				StatusDescription: "test description",
+			}
+
+			_, err := ec.EvaluatePolicy(ctx, evaluator)
+			require.NoError(t, err)
+
+			require.NotNil(t, ec.Status, "status should be set")
+			assert.Equal(t, test.expectedState, *ec.Status.State, "expected state %q, got %q", test.expectedState, *ec.Status.State)
+		})
+	}
+}
+
+func TestEvalContext_EvaluatePolicy_PendingAsFailure_NilConfig(t *testing.T) {
+	ctx := context.Background()
+
+	prctx := &pulltest.Context{
+		OwnerValue:     "testowner",
+		RepoValue:      "testrepo",
+		NumberValue:    1,
+		HeadSHAValue:   "abc123",
+		BranchBaseName: "main",
+	}
+
+	// Test with nil Config in FetchedConfig
+	ec := &EvalContext{
+		Options: &PullEvaluationOptions{
+			StatusCheckContext: "policy-bot",
+			PendingAsFailure:   true,
+		},
+		PublicURL:   "http://localhost:8080",
+		PullContext: prctx,
+		Config: FetchedConfig{
+			Config: nil, // nil config
+			Source: "test",
+			Path:   ".policy.yml",
+		},
+		SkipPostStatus: true,
+	}
+
+	evaluator := &staticEvaluator{
+		Status:            common.StatusPending,
+		StatusDescription: "test description",
+	}
+
+	_, err := ec.EvaluatePolicy(ctx, evaluator)
+	require.NoError(t, err)
+
+	require.NotNil(t, ec.Status, "status should be set")
+	assert.Equal(t, "failure", *ec.Status.State, "server option should be used when policy config is nil")
+}

server/handler/eval_options.go

@@ -60,6 +60,11 @@ type PullEvaluationOptions struct {
 	// context behaviour, and will be removed in 2.0
 	PostInsecureStatusChecks bool `yaml:"post_insecure_status_checks"`
 
+	// PendingAsFailure reports "failure" instead of "pending" for PRs that
+	// don't meet approval requirements. This is useful for tools like Kodiak
+	// that require definitive pass/fail status checks.
+	PendingAsFailure bool `yaml:"pending_as_failure"`
+
 	// IgnoreEditedComments enables ignoring comments that have been edited when evaluating approval rules.
 	// This provides a server-side option to ignore edited comments across all rules.
 	IgnoreEditedComments *bool `yaml:"ignore_edited_comments"`
@@ -118,6 +123,7 @@ func (p *PullEvaluationOptions) SetValuesFromEnv(prefix string) {
 	setBoolFromEnv("EXPAND_REQUIRED_REVIEWERS", prefix, &p.ExpandRequiredReviewers)
 	setBoolFromEnv("STRICT_REVIEW_DISMISSAL", prefix, &p.StrictReviewDismissal)
 	setBoolFromEnv("POST_INSECURE_STATUS_CHECKS", prefix, &p.PostInsecureStatusChecks)
+	setBoolFromEnv("PENDING_AS_FAILURE", prefix, &p.PendingAsFailure)
 	setBoolPtrFromEnv("IGNORE_EDITED_COMMENTS", prefix, &p.IgnoreEditedComments)
 
 	p.setApprovalDefaultsFromEnv(prefix + "APPROVAL_DEFAULTS_")

server/handler/eval_options_test.go

@@ -77,6 +77,12 @@ func TestPullEvaluationOptions_SetValuesFromEnv(t *testing.T) {
 				opts.PostInsecureStatusChecks = true
 			},
 		},
+		"PendingAsFailure": {
+			Env: map[string]string{"PEO_PENDING_AS_FAILURE": "true"},
+			SetExpected: func(opts *PullEvaluationOptions) {
+				opts.PendingAsFailure = true
+			},
+		},
 		"IgnoreEditedComments": {
 			Env: map[string]string{"PEO_IGNORE_EDITED_COMMENTS": "true"},
 			SetExpected: func(opts *PullEvaluationOptions) {