Skip to content

Commit 64f3e37

Browse files
enoodleenoodle
authored
test(scheduler): benchmark reclaim fit-error retention across full cycle (#1920)
Signed-off-by: Erez Freiberger <enoodle@gmail.com>
1 parent 900fe5f commit 64f3e37

2 files changed

Lines changed: 116 additions & 19 deletions

File tree

‎pkg/scheduler/actions/integration_tests/reclaim/reclaim_benchmark_test.go‎

Lines changed: 93 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package reclaim
66
import (
77
"flag"
88
"fmt"
9+
"runtime"
910
"testing"
1011
"time"
1112

@@ -15,7 +16,11 @@ import (
1516

1617
kaiv1 "github.com/kai-scheduler/KAI-scheduler/pkg/apis/kai/v1"
1718
commonconstants "github.com/kai-scheduler/KAI-scheduler/pkg/common/constants"
19+
"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/actions/allocate"
20+
"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/actions/consolidation"
21+
"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/actions/preempt"
1822
"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/actions/reclaim"
23+
"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/actions/stalegangeviction"
1924
"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/api/common_info"
2025
"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/api/pod_status"
2126
"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/framework"
@@ -34,6 +39,8 @@ var reclaimLargeJobNodeLocalGreedyBudget = flag.String(
3439
"optional NodeLocalGreedy generator budget override for BenchmarkReclaimLargeJobs",
3540
)
3641

42+
var manySingleGPUJobsBenchmarkKeepAlive *framework.Session
43+
3744
func init() {
3845
test_utils.InitTestingInfrastructure()
3946
}
@@ -62,46 +69,116 @@ func BenchmarkReclaimLargeJobs_1000Node(b *testing.B) {
6269
benchmarkReclaimLargeJobs(b, 1000)
6370
}
6471

65-
func BenchmarkReclaimManySingleGPUJobs_10Node(b *testing.B) {
66-
benchmarkReclaimManySingleGPUJobs(b, 10)
72+
func BenchmarkReclaimManySingleGPUJobsFullCycle_10Node(b *testing.B) {
73+
benchmarkReclaimManySingleGPUJobsFullCycle(b, 10)
6774
}
6875

69-
func BenchmarkReclaimManySingleGPUJobs_50Node(b *testing.B) {
70-
benchmarkReclaimManySingleGPUJobs(b, 50)
76+
func BenchmarkReclaimManySingleGPUJobsFullCycle_50Node(b *testing.B) {
77+
benchmarkReclaimManySingleGPUJobsFullCycle(b, 50)
7178
}
7279

73-
func BenchmarkReclaimManySingleGPUJobs_100Node(b *testing.B) {
74-
benchmarkReclaimManySingleGPUJobs(b, 100)
80+
func BenchmarkReclaimManySingleGPUJobsFullCycle_100Node(b *testing.B) {
81+
benchmarkReclaimManySingleGPUJobsFullCycle(b, 100)
7582
}
7683

77-
func BenchmarkReclaimManySingleGPUJobs_200Node(b *testing.B) {
78-
benchmarkReclaimManySingleGPUJobs(b, 200)
84+
func BenchmarkReclaimManySingleGPUJobsFullCycle_200Node(b *testing.B) {
85+
benchmarkReclaimManySingleGPUJobsFullCycle(b, 200)
7986
}
8087

81-
func BenchmarkReclaimManySingleGPUJobs_500Node(b *testing.B) {
82-
benchmarkReclaimManySingleGPUJobs(b, 500)
88+
func BenchmarkReclaimManySingleGPUJobsFullCycle_500Node(b *testing.B) {
89+
benchmarkReclaimManySingleGPUJobsFullCycleWithParams(b, defaultManySingleGPUJobsReclaimParams(500), true)
8390
}
8491

85-
func BenchmarkReclaimManySingleGPUJobsWithMinRuntime_500Node(b *testing.B) {
86-
benchmarkReclaimManySingleGPUJobsWithParams(b, manySingleGPUJobsReclaimParamsWithMinRuntime(500))
92+
func BenchmarkReclaimManySingleGPUJobsFullCycleWithMinRuntime_500Node(b *testing.B) {
93+
benchmarkReclaimManySingleGPUJobsFullCycleWithParams(b, manySingleGPUJobsReclaimParamsWithMinRuntime(500), false)
8794
}
8895

89-
func benchmarkReclaimManySingleGPUJobs(b *testing.B, numNodes int) {
90-
benchmarkReclaimManySingleGPUJobsWithParams(b, defaultManySingleGPUJobsReclaimParams(numNodes))
96+
func benchmarkReclaimManySingleGPUJobsFullCycle(b *testing.B, numNodes int) {
97+
benchmarkReclaimManySingleGPUJobsFullCycleWithParams(b, defaultManySingleGPUJobsReclaimParams(numNodes), false)
9198
}
9299

93-
func benchmarkReclaimManySingleGPUJobsWithParams(b *testing.B, params manySingleGPUJobsReclaimParams) {
100+
func benchmarkReclaimManySingleGPUJobsFullCycleWithParams(
101+
b *testing.B,
102+
params manySingleGPUJobsReclaimParams,
103+
measureLiveHeap bool,
104+
) {
94105
defer gock.Off()
95106

96107
topology := buildManySingleGPUJobsReclaimTopology(params)
108+
actions := manySingleGPUJobsSchedulingCycleActions()
97109
b.ReportAllocs()
110+
var heapAfterAllocate uint64
111+
var heapAfterCycle uint64
112+
var fitErrorTasks int
98113

99114
for b.Loop() {
115+
var before runtime.MemStats
116+
if measureLiveHeap {
117+
b.StopTimer()
118+
runtime.GC()
119+
runtime.ReadMemStats(&before)
120+
b.StartTimer()
121+
}
122+
100123
ctrl := gomock.NewController(b)
101124
ssn := test_utils.BuildSession(topology, ctrl)
102-
reclaim.New().Execute(ssn)
125+
for actionIndex, action := range actions {
126+
action.Execute(ssn)
127+
if measureLiveHeap && actionIndex == 0 {
128+
fitErrorTasks += countManySingleGPUFitErrorTasks(ssn)
129+
manySingleGPUJobsBenchmarkKeepAlive = ssn
130+
b.StopTimer()
131+
runtime.GC()
132+
var after runtime.MemStats
133+
runtime.ReadMemStats(&after)
134+
heapAfterAllocate += heapDelta(before.HeapAlloc, after.HeapAlloc)
135+
b.StartTimer()
136+
}
137+
}
138+
if measureLiveHeap {
139+
manySingleGPUJobsBenchmarkKeepAlive = ssn
140+
b.StopTimer()
141+
runtime.GC()
142+
var after runtime.MemStats
143+
runtime.ReadMemStats(&after)
144+
heapAfterCycle += heapDelta(before.HeapAlloc, after.HeapAlloc)
145+
b.StartTimer()
146+
}
103147
ctrl.Finish()
104148
}
149+
b.StopTimer()
150+
b.ReportMetric(1, "full_cycles/op")
151+
if measureLiveHeap {
152+
iterations := uint64(b.N)
153+
b.ReportMetric(float64(fitErrorTasks)/float64(iterations), "fit_error_tasks_after_allocate/op")
154+
b.ReportMetric(float64(heapAfterAllocate/iterations), "heap_live_after_allocate_bytes/op")
155+
b.ReportMetric(float64(heapAfterCycle/iterations), "heap_live_after_cycle_bytes/op")
156+
}
157+
}
158+
159+
func manySingleGPUJobsSchedulingCycleActions() []framework.Action {
160+
return []framework.Action{
161+
allocate.New(),
162+
consolidation.New(),
163+
reclaim.New(),
164+
preempt.New(),
165+
stalegangeviction.New(),
166+
}
167+
}
168+
169+
func countManySingleGPUFitErrorTasks(ssn *framework.Session) int {
170+
fitErrorTasks := 0
171+
for _, job := range ssn.ClusterInfo.PodGroupInfos {
172+
fitErrorTasks += len(job.TasksFitErrors)
173+
}
174+
return fitErrorTasks
175+
}
176+
177+
func heapDelta(before, after uint64) uint64 {
178+
if after <= before {
179+
return 0
180+
}
181+
return after - before
105182
}
106183

107184
func benchmarkReclaimLargeJobs(b *testing.B, numNodes int) {

‎pkg/scheduler/actions/integration_tests/reclaim/reclaim_many_single_gpu_topology_test.go‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ package reclaim
55

66
import (
77
"fmt"
8+
"reflect"
89
"strings"
910
"testing"
1011
"time"
@@ -13,7 +14,6 @@ import (
1314
"gopkg.in/h2non/gock.v1"
1415
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1516

16-
"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/actions/reclaim"
1717
"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/api/common_info"
1818
"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/api/pod_status"
1919
"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/constants"
@@ -23,6 +23,19 @@ import (
2323
"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/test_utils/tasks_fake"
2424
)
2525

26+
func TestManySingleGPUJobsSchedulingCycleActions(t *testing.T) {
27+
actions := manySingleGPUJobsSchedulingCycleActions()
28+
actionNames := make([]string, 0, len(actions))
29+
for _, action := range actions {
30+
actionNames = append(actionNames, string(action.Name()))
31+
}
32+
33+
expected := []string{"allocate", "consolidation", "reclaim", "preempt", "stalegangeviction"}
34+
if !reflect.DeepEqual(actionNames, expected) {
35+
t.Fatalf("scheduling cycle actions = %v, want %v", actionNames, expected)
36+
}
37+
}
38+
2639
type manySingleGPUJobsReclaimParams struct {
2740
NumNodes int
2841
GPUsPerNode int
@@ -55,9 +68,16 @@ func TestManySingleGPUJobsReclaimTopology(t *testing.T) {
5568
onJobSolutionStartCalls++
5669
})
5770

58-
reclaim.New().Execute(ssn)
59-
6071
expectedTasks := params.NumNodes * params.GPUsPerNode
72+
actions := manySingleGPUJobsSchedulingCycleActions()
73+
actions[0].Execute(ssn)
74+
if fitErrorTasks := countManySingleGPUFitErrorTasks(ssn); fitErrorTasks != expectedTasks {
75+
t.Fatalf("expected %d tasks with fit errors after allocate, got %d", expectedTasks, fitErrorTasks)
76+
}
77+
for _, action := range actions[1:] {
78+
action.Execute(ssn)
79+
}
80+
6181
if onJobSolutionStartCalls == 0 {
6282
t.Fatal("expected reclaim to attempt solving pending jobs")
6383
}

0 commit comments

Comments
 (0)