Skip to content

Commit 5f6fa46

Browse files
jtechappsjtechapps
committed
Add session affinity perf configurations and summary
add router configurations and inference-perf job for performance testing session affinity plugin with/without load aware plugin, with/without active-active scaling, and P/D disaggregation. add markdown files that summarize the findings Signed-off-by: Jacob Murry <jacobmurry@google.com>
1 parent 99298ae commit 5f6fa46

11 files changed

Lines changed: 701 additions & 1 deletion

‎test/perf/config/router-configs/load-aware-session-affinity-2rep.yaml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# Router configuration that combines load-aware-scorer with session-affinity-filter and passthrough-parser.
2+
# Configured with 2 active-active EPP replicas for horizontal scaling benchmark.
3+
router:
4+
extraServicePorts:
5+
- name: http
6+
port: 80
7+
protocol: TCP
8+
targetPort: 8081
9+
10+
epp:
11+
replicas: 2
12+
image:
13+
registry: ghcr.io/llm-d
14+
repository: llm-d-router-endpoint-picker
15+
tag: main
16+
pullPolicy: Always
17+
flags:
18+
v: 4
19+
enable-pprof: "true"
20+
tracing: "false"
21+
ha-enable-leader-election: "false"
22+
pluginsConfigFile: "optimized-baseline-plugins.yaml"
23+
pluginsCustomConfig:
24+
optimized-baseline-plugins.yaml: |
25+
apiVersion: llm-d.ai/v1alpha1
26+
kind: EndpointPickerConfig
27+
plugins:
28+
- type: passthrough-parser
29+
- type: session-affinity-filter
30+
parameters:
31+
headerName: x-session-token
32+
- type: load-aware-scorer
33+
parameters:
34+
threshold: 128
35+
- type: weighted-random-picker
36+
requestHandler:
37+
parsers:
38+
- pluginRef: passthrough-parser
39+
schedulingProfiles:
40+
- name: default
41+
plugins:
42+
- pluginRef: session-affinity-filter
43+
- pluginRef: load-aware-scorer
44+
weight: 5
45+
- pluginRef: weighted-random-picker
46+
47+
inferencePool:
48+
failureMode: "FailOpen"
49+
50+
modelServers:
51+
protocol: http
52+
53+
proxy:
54+
args:
55+
- "--service-node"
56+
- "envoy-sidecar"
57+
- "--log-level"
58+
- "warn"
59+
- "--concurrency"
60+
- "32"
61+
- "--drain-strategy"
62+
- "immediate"
63+
- "--drain-time-s"
64+
- "60"
65+
- "-c"
66+
- "/etc/envoy/envoy.yaml"
67+
resources:
68+
requests:
69+
cpu: "4"
70+
memory: 8Gi
71+
limits:
72+
memory: 16Gi
73+
74+
tracing:
75+
otelExporterEndpoint: "http://localhost:4317"
76+
sampling:
77+
sampler: "parentbased_traceidratio"
78+
samplerArg: "0.1"

‎test/perf/config/router-configs/load-aware-session-affinity-3rep.yaml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# Router configuration that combines load-aware-scorer with session-affinity-filter and passthrough-parser.
2+
# Configured with 3 active-active EPP replicas for horizontal scaling benchmark.
3+
router:
4+
extraServicePorts:
5+
- name: http
6+
port: 80
7+
protocol: TCP
8+
targetPort: 8081
9+
10+
epp:
11+
replicas: 3
12+
image:
13+
registry: ghcr.io/llm-d
14+
repository: llm-d-router-endpoint-picker
15+
tag: main
16+
pullPolicy: Always
17+
flags:
18+
v: 4
19+
enable-pprof: "true"
20+
tracing: "false"
21+
ha-enable-leader-election: "false"
22+
pluginsConfigFile: "optimized-baseline-plugins.yaml"
23+
pluginsCustomConfig:
24+
optimized-baseline-plugins.yaml: |
25+
apiVersion: llm-d.ai/v1alpha1
26+
kind: EndpointPickerConfig
27+
plugins:
28+
- type: passthrough-parser
29+
- type: session-affinity-filter
30+
parameters:
31+
headerName: x-session-token
32+
- type: load-aware-scorer
33+
parameters:
34+
threshold: 128
35+
- type: weighted-random-picker
36+
requestHandler:
37+
parsers:
38+
- pluginRef: passthrough-parser
39+
schedulingProfiles:
40+
- name: default
41+
plugins:
42+
- pluginRef: session-affinity-filter
43+
- pluginRef: load-aware-scorer
44+
weight: 5
45+
- pluginRef: weighted-random-picker
46+
47+
inferencePool:
48+
failureMode: "FailOpen"
49+
50+
modelServers:
51+
protocol: http
52+
53+
proxy:
54+
args:
55+
- "--service-node"
56+
- "envoy-sidecar"
57+
- "--log-level"
58+
- "warn"
59+
- "--concurrency"
60+
- "32"
61+
- "--drain-strategy"
62+
- "immediate"
63+
- "--drain-time-s"
64+
- "60"
65+
- "-c"
66+
- "/etc/envoy/envoy.yaml"
67+
resources:
68+
requests:
69+
cpu: "4"
70+
memory: 8Gi
71+
limits:
72+
memory: 16Gi
73+
74+
tracing:
75+
otelExporterEndpoint: "http://localhost:4317"
76+
sampling:
77+
sampler: "parentbased_traceidratio"
78+
samplerArg: "0.1"

‎test/perf/config/router-configs/load-aware-session-affinity.yaml‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# Router configuration that combines load-aware-scorer with session-affinity-filter and passthrough-parser.
2+
router:
3+
extraServicePorts:
4+
- name: http
5+
port: 80
6+
protocol: TCP
7+
targetPort: 8081
8+
9+
epp:
10+
replicas: 1
11+
image:
12+
registry: ghcr.io/llm-d
13+
repository: llm-d-router-endpoint-picker
14+
tag: main
15+
pullPolicy: Always
16+
flags:
17+
v: 4
18+
enable-pprof: "true"
19+
tracing: "false"
20+
pluginsConfigFile: "optimized-baseline-plugins.yaml"
21+
pluginsCustomConfig:
22+
optimized-baseline-plugins.yaml: |
23+
apiVersion: llm-d.ai/v1alpha1
24+
kind: EndpointPickerConfig
25+
plugins:
26+
- type: passthrough-parser
27+
- type: session-affinity-filter
28+
parameters:
29+
headerName: x-session-token
30+
- type: load-aware-scorer
31+
parameters:
32+
threshold: 128
33+
- type: weighted-random-picker
34+
requestHandler:
35+
parsers:
36+
- pluginRef: passthrough-parser
37+
schedulingProfiles:
38+
- name: default
39+
plugins:
40+
- pluginRef: session-affinity-filter
41+
- pluginRef: load-aware-scorer
42+
weight: 5
43+
- pluginRef: weighted-random-picker
44+
45+
inferencePool:
46+
failureMode: "FailOpen"
47+
48+
modelServers:
49+
protocol: http
50+
51+
proxy:
52+
args:
53+
- "--service-node"
54+
- "envoy-sidecar"
55+
- "--log-level"
56+
- "warn"
57+
- "--concurrency"
58+
- "32"
59+
- "--drain-strategy"
60+
- "immediate"
61+
- "--drain-time-s"
62+
- "60"
63+
- "-c"
64+
- "/etc/envoy/envoy.yaml"
65+
resources:
66+
requests:
67+
cpu: "4"
68+
memory: 8Gi
69+
limits:
70+
memory: 16Gi
71+
72+
tracing:
73+
otelExporterEndpoint: "http://localhost:4317"
74+
sampling:
75+
sampler: "parentbased_traceidratio"
76+
samplerArg: "0.1"

‎test/perf/config/router-configs/optimized-baseline.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ router:
5656
- "--log-level"
5757
- "warn"
5858
- "--concurrency"
59-
- "8"
59+
- "32"
6060
- "--drain-strategy"
6161
- "immediate"
6262
- "--drain-time-s"

‎test/perf/config/router-configs/pd-session-affinity.yaml‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
# Router configuration for Prefill/Decode (P/D) disaggregation with session affinity using llm-d-sim.
2+
router:
3+
extraServicePorts:
4+
- name: http
5+
port: 80
6+
protocol: TCP
7+
targetPort: 8081
8+
9+
epp:
10+
replicas: 1
11+
image:
12+
registry: ghcr.io/llm-d
13+
repository: llm-d-router-endpoint-picker
14+
tag: main
15+
pullPolicy: Always
16+
flags:
17+
v: 4
18+
enable-pprof: "true"
19+
tracing: "false"
20+
ha-enable-leader-election: "false"
21+
pluginsConfigFile: "optimized-baseline-plugins.yaml"
22+
pluginsCustomConfig:
23+
optimized-baseline-plugins.yaml: |
24+
apiVersion: llm-d.ai/v1alpha1
25+
kind: EndpointPickerConfig
26+
plugins:
27+
- type: passthrough-parser
28+
- type: always-disagg-pd-decider
29+
- type: disagg-profile-handler
30+
parameters:
31+
deciders:
32+
prefill: always-disagg-pd-decider
33+
- type: session-affinity-filter
34+
name: session-affinity-decode
35+
parameters:
36+
headerName: x-session-token
37+
- type: session-affinity-filter
38+
name: session-affinity-prefill
39+
parameters:
40+
headerName: x-session-token-prefill
41+
profileName: prefill
42+
- type: random-picker
43+
requestHandler:
44+
parsers:
45+
- pluginRef: passthrough-parser
46+
schedulingProfiles:
47+
- name: prefill
48+
plugins:
49+
- pluginRef: session-affinity-prefill
50+
- pluginRef: random-picker
51+
- name: decode
52+
plugins:
53+
- pluginRef: session-affinity-decode
54+
- pluginRef: random-picker
55+
56+
inferencePool:
57+
failureMode: "FailOpen"
58+
59+
modelServers:
60+
protocol: http
61+
62+
proxy:
63+
args:
64+
- "--service-node"
65+
- "envoy-sidecar"
66+
- "--log-level"
67+
- "warn"
68+
- "--concurrency"
69+
- "32"
70+
- "--drain-strategy"
71+
- "immediate"
72+
- "--drain-time-s"
73+
- "60"
74+
- "-c"
75+
- "/etc/envoy/envoy.yaml"
76+
resources:
77+
requests:
78+
cpu: "4"
79+
memory: 8Gi
80+
limits:
81+
memory: 16Gi
82+
83+
tracing:
84+
otelExporterEndpoint: "http://localhost:4317"
85+
sampling:
86+
sampler: "parentbased_traceidratio"
87+
samplerArg: "0.1"

0 commit comments

Comments
 (0)