feat: enable the flowControl feature gate by default

apix/config/v1alpha1/endpointpickerconfig_types.go

@@ -34,8 +34,9 @@ type EndpointPickerConfig struct {
 	metav1.TypeMeta `json:",inline"`
 
 	// +optional
-	// FeatureGates is a set of flags that enable various experimental features with the EPP.
-	// If omitted none of these experimental features will be enabled.
+	// FeatureGates is a set of flags that toggle optional EPP features. Each entry is a gate name,
+	// optionally suffixed with "=true" or "=false" (a bare name means "=true"). Gates carry
+	// per-gate defaults that apply when omitted; some (such as "flowControl") default to enabled.
 	FeatureGates FeatureGates `json:"featureGates,omitempty"`
 
 	// +required
@@ -186,7 +187,8 @@ func (sp SchedulingPlugin) String() string {
 	return "{" + strings.Join(parts, ", ") + "}"
 }
 
-// FeatureGates is a set of flags that enable various experimental features with the EPP
+// FeatureGates is a set of flags that toggle optional EPP features ("name", "name=true", or
+// "name=false"); omitted gates use their registered defaults.
 type FeatureGates []string
 
 func (fg FeatureGates) String() string {

cmd/epp/runner/feature_gate_test.go

@@ -127,6 +127,8 @@ featureGates:
 				wantEnabled = *tc.wantEnabled
 				require.Equal(t, wantEnabled, r.featureGates[flowcontrol.FeatureGate],
 					"the loader should honor the explicit featureGates stanza")
+			} else {
+				require.True(t, wantEnabled, "the flowControl gate is registered as enabled by default (#2104)")
 			}
 
 			ds := datastore.NewDatastore(ctx, r.setupMetricsCollection(opts))

cmd/epp/runner/runner.go

@@ -147,7 +147,8 @@ import (
 const (
 	// enableExperimentalFlowControlLayer defines the environment variable used as a feature flag for the pluggable flow
 	// control layer.
-	// DEPRECATION NOTICE - this env var will be removed in the next version as we switch to configuring the EPP using FeatureGates in the config file.
+	// DEPRECATION NOTICE - the flowControl feature gate is enabled by default, so this env var is a no-op; it will be
+	// removed in a future release. Use featureGates in the config file (e.g., "flowControl=false") instead.
 	enableExperimentalFlowControlLayer = "ENABLE_EXPERIMENTAL_FLOW_CONTROL_LAYER"
 )
 
@@ -685,7 +686,7 @@ func (r *Runner) parseConfigurationPhaseOne(ctx context.Context, opts *runserver
 		}
 	}
 
-	loader.RegisterFeatureGate(flowcontrol.FeatureGate, false)
+	loader.RegisterFeatureGate(flowcontrol.FeatureGate, true)
 	loader.RegisterFeatureGate(runserver.HAPopulateNonLeaderDatastoreFeatureGate, true)
 
 	r.registerInTreePlugins()
@@ -767,7 +768,7 @@ func (r *Runner) parseConfigurationPhaseTwo(ctx context.Context, rawConfig *conf
 
 func applyDeprecatedEnvFeatureGate(envVar, featureName, featureGate string, rawConfig *configapi.EndpointPickerConfig) {
 	if _, ok := os.LookupEnv(envVar); ok {
-		setupLog.Info(fmt.Sprintf("Enabling the experimental %s using environment variables is deprecated and will be removed in next version", featureName))
+		setupLog.Info(fmt.Sprintf("The %s environment variable (current value %q) is deprecated and will be removed in a future release; the %s is enabled by default, so this variable no longer disables it. To disable it, set featureGates: [%q] in the config file", envVar, os.Getenv(envVar), featureName, featureGate+"=false"))
 		if env.GetEnvBool(envVar, false, setupLog) {
 			if rawConfig.FeatureGates == nil {
 				rawConfig.FeatureGates = make(configapi.FeatureGates, 0)
@@ -892,13 +893,13 @@ func (r *Runner) initAdmissionControl(
 	endpointCandidates contracts.EndpointCandidates,
 ) (contracts.EndpointCandidates, requestcontrol.AdmissionController, contracts.PriorityBandControlPlane) {
 	if !r.featureGates[flowcontrol.FeatureGate] {
-		setupLog.Info("Experimental Flow Control layer is disabled, using legacy admission control")
+		setupLog.Info("Flow Control layer is disabled via the flowControl feature gate, using legacy admission control")
 		return endpointCandidates,
 			requestcontrol.NewLegacyAdmissionController(eppConfig.SaturationDetector, endpointCandidates),
 			nil
 	}
 	endpointCandidates = requestcontrol.NewCachedEndpointCandidates(ctx, endpointCandidates, 50*time.Millisecond)
-	setupLog.Info("Initializing experimental Flow Control layer")
+	setupLog.Info("Initializing Flow Control layer")
 	registry := fcregistry.NewFlowRegistry(eppConfig.FlowControlConfig.Registry, setupLog)
 	fc := fccontroller.NewFlowController(
 		ctx,

docs/architecture.md

@@ -195,6 +195,8 @@ RequestHandler:
 - When no parsers are configured, `openai-parser`, `anthropic-parser`, and `vllmhttp-parser` are used.
 
 FlowControl:
+- The flow control admission layer itself is on by default (the `flowControl` feature gate);
+  disable it with `featureGates: ["flowControl=false"]`.
 - `fcfs-ordering-policy`, `global-strict-fairness-policy`, and `static-usage-limit-policy` are configured when absent.
 - `utilization-detector` is configured as the saturation detector when none is set.
 

docs/metrics.md

@@ -116,7 +116,7 @@ Unlabelled.
 
 | Name | Type | Notes |
 |---|---|---|
-| `request_processing_duration_seconds` | Histogram | Time from request receipt until the request body has been handled. Includes admission control, so under the flow control feature gate this covers queue wait; `flow_control_request_queue_duration_seconds` separates it out. |
+| `request_processing_duration_seconds` | Histogram | Time from request receipt until the request body has been handled. Includes admission control, so with flow control on (the default) this covers queue wait; `flow_control_request_queue_duration_seconds` separates it out. |
 | `response_processing_duration_seconds` | Histogram | Sum of the per-chunk handler slices for a streamed response, so model-server generation time between chunks is excluded. For a non-streaming response, the interval from response headers to completion. |
 
 ### Plugin, info, and model rewrite
@@ -204,7 +204,8 @@ are under `llm_d_epp_`; each has a deprecated `llm_d_inference_scheduler_*` twin
 
 ### Flow control
 
-Exposed when the `flowControl` feature gate is enabled.
+Exposed when the `flowControl` feature gate is enabled, which is the default; opting out via
+`featureGates: ["flowControl=false"]` removes these series.
 
 #### `flow_control_request_queue_duration_seconds`
 

docs/operations.md

@@ -19,6 +19,15 @@ The EPP acts as the routing intelligence engine. Its resource usage scales prima
 #### Memory Allocation
 - **Base Memory**: EPP memory usage is relatively low and stable with small output token requests, but scales with the number of concurrent inflight requests.
 - **Inflight Requests Impact**: Memory usage increases with the number of concurrent inflight requests and the output (decode) token length.
+- **Flow Control Queues**: With flow control enabled (the default), requests that cannot dispatch
+  under saturation are buffered in EPP memory, including their request bodies. The buffered volume
+  is bounded per priority band by `priorityBands[].maxRequests` (default 5000) and `maxBytes`
+  (default 1G), which `defaultPriorityBand` sets as a template for bands you do not list; budget for
+  the sum of the per-band `maxBytes` limits of the priority levels your traffic actually uses, on top
+  of the inflight-request sizing above. The global `flowControl.maxRequests` / `maxBytes` caps
+  default to unlimited, so set a global `maxBytes` under the container memory limit: at the per-band
+  default, a handful of bands clears the sizing guidance below before any band cap engages. Lower
+  these limits (or set a shorter `defaultRequestTTL`) to trade queueing for earlier shedding.
 - **Sizing Guidelines**:
   - For a request rate of 50 to 100 requests/second with 1k output tokens, EPP requires between **4 GiB and 6 GiB** of memory.
   - For workloads with longer output lengths (such as 5k output tokens), memory usage can reach **20+ GiB** due to the accumulation of state for concurrent inflight requests.
@@ -37,6 +46,10 @@ The EPP's scaling behavior and effectiveness are highly dependent on the configu
   | 3 | 2.7x |
   | 4 | 3.5x |
 
+  - **Note (Flow Control)**: Flow control state (queues, fairness accounting, and the saturation
+    view) is per replica and not shared. In Active-Active mode, priority and fairness are enforced
+    only within each replica's share of the traffic, and per-band capacity limits apply per
+    replica, so the fleet-wide queued volume scales with the replica count.
   - **Warning (Prefix Routing)**: **Active-Active mode should be avoided when using approximate prefix routing.** Because EPP replicas do not share prefix state, each replica only has visibility into the prefix state of the requests it has individually handled. This partition of state significantly degrades prefix cache hit rates, making prefix caching highly inefficient.
   - For more technical details and context on EPP replica state sync and scaling limitations, see [Issue #1290](https://github.com/llm-d/llm-d-router/issues/1290).
 

pkg/epp/config/loader/configloader.go

@@ -192,6 +192,8 @@ func InstantiateAndConfigure(
 		if err != nil {
 			return nil, fmt.Errorf("failed to load flow control config: %w", err)
 		}
+	} else if flowControlSettingsConfigured(rawConfig.FlowControl) {
+		logger.Info("WARNING: the flowControl config section is set but the flowControl feature gate is disabled; its settings (other than saturationDetector) are ignored")
 	}
 
 	parserRegistry, err := buildParserRegistry(rawConfig.RequestHandler.Parsers, handle, logger)
@@ -217,6 +219,19 @@ func InstantiateAndConfigure(
 	}, nil
 }
 
+// flowControlSettingsConfigured reports whether the flowControl config section carries settings
+// beyond the saturation detector. The saturation detector is honored by the legacy admission path
+// even when the flowControl feature gate is disabled, so it alone does not indicate ignored
+// configuration.
+func flowControlSettingsConfigured(fc *configapi.FlowControlConfig) bool {
+	if fc == nil {
+		return false
+	}
+	return fc.MaxBytes != nil || fc.MaxRequests != nil || fc.DefaultRequestTTL != nil ||
+		fc.DefaultPriorityBand != nil || fc.DefaultNegativePriorityBand != nil ||
+		len(fc.PriorityBands) > 0 || fc.UsageLimitPolicyPluginRef != ""
+}
+
 func decodeRawConfig(configBytes []byte) (*configapi.EndpointPickerConfig, error) {
 	cfg := &configapi.EndpointPickerConfig{}
 	codecs := serializer.NewCodecFactory(scheme, serializer.EnableStrict)

pkg/epp/config/loader/configloader_test.go

@@ -80,7 +80,7 @@ func TestLoadRawConfiguration(t *testing.T) {
 
 	// Register known feature gates for validation.
 	RegisterFeatureGate(testFeatureGate, true)
-	RegisterFeatureGate(flowcontrol.FeatureGate, false)
+	RegisterFeatureGate(flowcontrol.FeatureGate, true)
 
 	queueScorerWeight := 2.0
 	kvCacheUtilizationScorerWeight := 2.0
@@ -189,7 +189,7 @@ func TestLoadRawConfiguration(t *testing.T) {
 			},
 			wantFeatures: map[string]bool{
 				testFeatureGate:         false,
-				flowcontrol.FeatureGate: false,
+				flowcontrol.FeatureGate: true,
 			},
 			wantErr:    false,
 			deprecated: false,
@@ -257,7 +257,7 @@ func TestLoadRawConfiguration(t *testing.T) {
 			},
 			wantFeatures: map[string]bool{
 				testFeatureGate:         true,
-				flowcontrol.FeatureGate: false,
+				flowcontrol.FeatureGate: true,
 			},
 			wantErr:    false,
 			deprecated: false,
@@ -444,7 +444,7 @@ func TestInstantiateAndConfigure(t *testing.T) {
 	registerTestPlugins(t)
 
 	RegisterFeatureGate(testFeatureGate, true)
-	RegisterFeatureGate(flowcontrol.FeatureGate, false)
+	RegisterFeatureGate(flowcontrol.FeatureGate, true)
 
 	tests := []struct {
 		name       string
@@ -561,7 +561,7 @@ func TestInstantiateAndConfigure(t *testing.T) {
 			},
 		},
 		{
-			name:       "Ignored - Flow Control Config Present but FeatureGate Missing",
+			name:       "Ignored - Flow Control Config Present but FeatureGate Disabled",
 			configText: successflowControlConfigDisabledText,
 			wantErr:    false,
 			validate: func(t *testing.T, handle fwkplugin.Handle, rawCfg *configapi.EndpointPickerConfig, cfg *config.Config) {
@@ -850,6 +850,71 @@ func TestInstantiateAndConfigure(t *testing.T) {
 	}
 }
 
+// TestFlowControlConfigIgnoredWarning verifies that a flowControl config section combined with a
+// disabled flowControl feature gate logs a warning that the settings are ignored, and that the
+// warning stays silent otherwise. The silent cases carry the weight here: ensureSaturationDetector
+// populates FlowControl for every config, so a predicate that ignored its saturationDetector
+// exclusion would warn at every legacy-path startup.
+func TestFlowControlConfigIgnoredWarning(t *testing.T) {
+	// Not parallel because it modifies the global plugin registry.
+	registerTestPlugins(t)
+	RegisterFeatureGate(flowcontrol.FeatureGate, true)
+
+	testCases := []struct {
+		name        string
+		configText  string
+		gateEnabled bool
+		wantWarn    bool
+	}{
+		{
+			name:       "settings ignored under an explicit opt-out",
+			configText: successflowControlConfigDisabledText,
+			wantWarn:   true,
+		},
+		{
+			name:       "opt-out with no flowControl section",
+			configText: successFlowControlDisabledNoSectionText,
+		},
+		{
+			name:       "opt-out with only a saturation detector",
+			configText: successFlowControlDisabledSaturationDetectorText,
+		},
+		{
+			name:        "settings honored when the gate is on",
+			configText:  successFlowControlConfigText,
+			gateEnabled: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			writer := &strings.Builder{}
+			logger := logging.NewTestLoggerWithWriter(writer)
+
+			rawConfig, _, err := LoadRawConfig([]byte(tc.configText), logger)
+			require.NoError(t, err)
+
+			handle := testutils.NewTestHandle(context.Background())
+			cfg, err := InstantiateAndConfigure(rawConfig, handle, logger)
+			require.NoError(t, err)
+
+			if tc.gateEnabled {
+				require.NotNil(t, cfg.FlowControlConfig, "flow control config should be built when the gate is on")
+			} else {
+				require.Nil(t, cfg.FlowControlConfig, "flow control config should not be built when the gate is disabled")
+			}
+
+			if tc.wantWarn {
+				require.Contains(t, writer.String(), "flowControl feature gate is disabled",
+					"the ignored flowControl section should be called out in the logs")
+			} else {
+				require.NotContains(t, writer.String(), "flowControl feature gate is disabled",
+					"nothing is being ignored, so the warning should stay silent")
+			}
+		})
+	}
+}
+
 // TestBuildDataLayerConfigEmptySourcesWarning verifies that an empty sources list
 // logs a warning but does not return an error.
 func TestBuildDataLayerConfigEmptySourcesWarning(t *testing.T) {

pkg/epp/config/loader/testdata_test.go

@@ -274,11 +274,43 @@ schedulingProfiles:
 - name: default
   plugins:
   - pluginRef: maxScore
-featureGates: [] # Explicitly empty
+featureGates: ["flowControl=false"] # Explicit opt-out; the gate is enabled by default.
 flowControl:
   maxBytes: "1024"
 `
 
+// successFlowControlDisabledNoSectionText is the plain legacy-path config: an explicit opt-out with
+// nothing for the loader to report as ignored.
+const successFlowControlDisabledNoSectionText = `
+apiVersion: llm-d.ai/v1alpha1
+kind: EndpointPickerConfig
+plugins:
+- name: maxScore
+  type: max-score-picker
+schedulingProfiles:
+- name: default
+  plugins:
+  - pluginRef: maxScore
+featureGates: ["flowControl=false"]
+`
+
+// successFlowControlDisabledSaturationDetectorText pairs an explicit opt-out with a saturation
+// detector, which the legacy admission path honors and so must not be reported as ignored.
+const successFlowControlDisabledSaturationDetectorText = `
+apiVersion: llm-d.ai/v1alpha1
+kind: EndpointPickerConfig
+plugins:
+- name: maxScore
+  type: max-score-picker
+schedulingProfiles:
+- name: default
+  plugins:
+  - pluginRef: maxScore
+featureGates: ["flowControl=false"]
+saturationDetector:
+  pluginRef: utilization-detector
+`
+
 // successComplexFlowControlConfigText tests that Flow Control configuration with custom plugins is correctly loaded.
 const successComplexFlowControlConfigText = `
 apiVersion: llm-d.ai/v1alpha1

pkg/epp/flowcontrol/README.md

@@ -30,6 +30,44 @@ between the Routing and Scheduling layers. It decides *if* and *when* a request,
 mechanism for managing diverse SLOs, ensuring fairness among competing workloads, and maintaining system stability under
 high load.
 
+### Enabling, disabling, and tuning
+
+Flow control is enabled by default via the `flowControl` feature gate. With it on, saturation no
+longer pushes excess requests into each model server's local queue: they wait in the EPP and
+dispatch by priority as capacity frees, with queue wait bounded by a TTL. Sheddable
+(negative-priority) requests buffer under the same TTL and capacity rules instead of being
+rejected immediately on saturation. To restore the legacy saturation-only admission behavior, opt
+out in the EPP config:
+
+```yaml
+featureGates: ["flowControl=false"]
+```
+
+Setting a `flowControl:` config section while the gate is disabled logs a warning: everything in it
+except `saturationDetector` (which the legacy admission path also uses) is ignored.
+
+Operationally, dispatch decisions depend on fresh endpoint metrics: saturation detection reads the
+model-server metrics that the EPP's own data layer scrapes from pods (this is independent of
+cluster monitoring such as Prometheus). Endpoints whose metrics are older than the detector's
+`metricsStalenessThreshold` count as fully saturated, so if the EPP loses its scrape path to all
+pods (network policy, metrics port change, a starved refresh loop), dispatch halts and queued
+requests eventually shed at their TTL. Keep the refresh interval comfortably inside the staleness
+threshold and monitor scrape health when running with flow control on.
+
+Tuning knobs, all under the `flowControl:` config section:
+
+* Per-band `maxRequests` / `maxBytes` — the shedding knobs. Lower them to reject excess load at the
+  queue boundary instead of buffering it (for example, to approximate the legacy immediate-shed
+  behavior for sheddable traffic).
+* `defaultRequestTTL` — the queue-wait budget, and the other way a request is shed. When the pool
+  has no endpoints the queue acts as a scale-from-zero waiting room and requests hold for the full
+  budget, so keep it under the client or gateway deadline unless you want requests to survive a cold
+  start.
+* The priority-holdback usage-limit policy — a gating knob, not a shedding one. It lowers the
+  admission ceiling for low-priority traffic as utilization rises, so that traffic waits in queue
+  rather than being rejected; it sheds only by way of the two limits above. Configure it via
+  `usageLimitPolicyPluginRef`.
+
 ### High Level Architecture
 
 The following diagram illustrates the high-level dependency model and request flow for the system. It shows how

test/e2e/configs_test.go

@@ -23,6 +23,15 @@ schedulingProfiles:
     weight: 2
 `
 
+// EPP configuration for the disruption tests that empty the pool.
+//
+// Flow control treats an endpointless pool as a scale-from-zero waiting room: requests hold in queue
+// for the full queue-wait budget rather than failing fast. These tests assert prompt failure instead
+// of cold-start queueing, so they shorten the budget to land the shed inside the client timeout.
+const disruptionConfig = simpleConfig + `flowControl:
+  defaultRequestTTL: 5s
+`
+
 // EPP configuration for running with P/D
 // Uses deprecated pd-profile-handler
 const deprecatedPdConfig = `apiVersion: llm-d.ai/v1alpha1

test/e2e/disruption_test.go

@@ -215,7 +215,7 @@ var _ = ginkgo.Describe("Disruption tests", func() {
 
 			modelServers := createModelServersDecode(1)
 
-			epp := createEndPointPicker(simpleConfig)
+			epp := createEndPointPicker(disruptionConfig)
 
 			_, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector, nsName)
 			gomega.Expect(decodePods).Should(gomega.HaveLen(1))
@@ -318,7 +318,7 @@ var _ = ginkgo.Describe("Disruption tests", func() {
 
 			modelServers := createModelServersDecode(1)
 
-			epp := createEndPointPicker(simpleConfig)
+			epp := createEndPointPicker(disruptionConfig)
 
 			ginkgo.By("Verifying requests succeed before disruption")
 			nsHdr, _, _ := runCompletion(simplePrompt, simModelName)