-
Notifications
You must be signed in to change notification settings - Fork 6
/
lab_usage_verbose.awk
executable file
·168 lines (146 loc) · 6.15 KB
/
lab_usage_verbose.awk
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
#!/bin/awk -f
@include "mycolors.awk"
BEGIN {
FS="[ :|=,]";
printf("| %14s |","Username");
printf(" %3s | %3s| %3s|", "N-R", "N-PD", "N-CG");
printf(" %3s | %3s| %3s|", "O-R", "O-PD", "O-CG");
printf(" %3s | %3s | %3s |", "R", "PD", "CG");
printf("\n");
printf("| %14s |","--------------");
printf(" %3s + %3s + %3s |", "---", "---", "---");
printf(" %3s + %3s + %3s |", "---", "---", "---");
printf(" %3s + %3s + %3s |", "---", "---", "---");
printf("\n");
printf("| %-68s |\n"," 'N': normal, 'O': overcap, none: total");
# printf("| %-68s |\n"," 'R': running, 'PD': pending, 'CG': interrupted");
printf("| %14s |","--------------");
printf(" %3s + %3s + %3s |", "---", "---", "---");
printf(" %3s + %3s + %3s |", "---", "---", "---");
printf(" %3s + %3s + %3s |", "---", "---", "---");
printf("\n");
}
{
if ($1 == "G>") {
# for overcap jobs, assign lab arbitrarily
if ($14 == "overcap"){
lab=user_to_lab[$2]
partition="overcap"
}
else {
lab=$13
partition="normal"
}
# GPU Counts
gpu_counts[lab][$2][$3]+=$12;
gpu_counts[lab][$2][$3,partition]+=$12;
gpu_counts[lab][$2]["R"]+=0;
gpu_counts[lab][$2]["PD"]+=0;
gpu_counts[lab][$2]["CG"]+=0;
gpu_counts[lab][$2]["R","normal"]+=0;
gpu_counts[lab][$2]["PD","normal"]+=0;
gpu_counts[lab][$2]["CG","normal"]+=0;
gpu_counts[lab][$2]["R","overcap"]+=0;
gpu_counts[lab][$2]["PD","overcap"]+=0;
gpu_counts[lab][$2]["CG","overcap"]+=0;
# CPU counts
cpu_counts[lab][$2][$3]+=$6;
cpu_counts[lab][$2][$3,partition]+=$6;
cpu_counts[lab][$2]["R"]+=0;
cpu_counts[lab][$2]["PD"]+=0;
cpu_counts[lab][$2]["CG"]+=0;
cpu_counts[lab][$2]["R","normal"]+=0;
cpu_counts[lab][$2]["PD","normal"]+=0;
cpu_counts[lab][$2]["CG","normal"]+=0;
cpu_counts[lab][$2]["R","overcap"]+=0;
cpu_counts[lab][$2]["PD","overcap"]+=0;
cpu_counts[lab][$2]["CG","overcap"]+=0;
labs_to_gpus_used[lab][$3]+=$12;
labs_to_cpus_used[lab][$3]+=$6;
lab_gpu_totals[lab][$3]+=$12;
lab_cpu_totals[lab][$3]+=$6;
lab_gpu_totals[lab][$3,partition]+=$12;
lab_cpu_totals[lab][$3,partition]+=$6;
} else {
if ($5 == "gres/gpu") {
labs_to_gpus[$1] += $6;
labs_to_cpus[$1] += $4;
} else {
if ($1 != "overcap") {
user_to_lab[$2]=$1;
}
}
}
}
END {
for (lab in labs_to_gpus) {
if (lab == "guest-lab") {
continue;
}
print_str = sprintf(\
"[ %d / %d / %d (Run/Sched/Total) GPUS]",
labs_to_gpus_used[lab]["R"],
labs_to_gpus_used[lab]["PD"],
labs_to_gpus[lab])
printf("| %14s = %-51s |\n", lab, print_str);
if (lab in gpu_counts) {
for (name in gpu_counts[lab]) {
printf("| %14s |",name);
# printf(" %3d |",gpu_counts[lab][name]["R","normal"]);
# printf(" %3d |",gpu_counts[lab][name]["PD","normal"]);
# printf(" %3d |",gpu_counts[lab][name]["CG","normal"]);
#
# printf(" %3d |",gpu_counts[lab][name]["R","overcap"]);
# printf(" %3d |",gpu_counts[lab][name]["PD","overcap"]);
# printf(" %3d |",gpu_counts[lab][name]["CG","overcap"]);
#
# printf(" %3d |",gpu_counts[lab][name]["R"]);
# printf(" %3d |",gpu_counts[lab][name]["PD"]);
# printf(" %3d |",gpu_counts[lab][name]["CG"]);
printf(" %s |",colour_int(gpu_counts[lab][name]["R","normal"]));
printf(" %s |",colour_int(gpu_counts[lab][name]["PD","normal"]));
printf(" %s |",colour_int(gpu_counts[lab][name]["CG","normal"]));
printf(" %s |",colour_int(gpu_counts[lab][name]["R","overcap"]));
printf(" %s |",colour_int(gpu_counts[lab][name]["PD","overcap"]));
printf(" %s |",colour_int(gpu_counts[lab][name]["CG","overcap"]));
printf(" %s |",colour_int(gpu_counts[lab][name]["R"]));
printf(" %s |",colour_int(gpu_counts[lab][name]["PD"]));
printf(" %s |",colour_int(gpu_counts[lab][name]["CG"]));
printf("\n");
}
}
printf("| %14s |","");
printf(" %3s + %3s + %3s |", "---", "---", "---");
printf(" %3s + %3s + %3s |", "---", "---", "---");
printf(" %3s + %3s + %3s |", "---", "---", "---");
printf("\n");
printf("| %14s :","totals");
printf(" %3d |",lab_gpu_totals[lab]["R","normal"]);
printf(" %3d |",lab_gpu_totals[lab]["PD","normal"]);
printf(" %3d |",lab_gpu_totals[lab]["CG","normal"]);
printf(" %3d |",lab_gpu_totals[lab]["R","overcap"]);
printf(" %3d |",lab_gpu_totals[lab]["PD","overcap"]);
printf(" %3d |",lab_gpu_totals[lab]["CG","overcap"]);
printf(" %3d |",lab_gpu_totals[lab]["R"]);
printf(" %3d |",lab_gpu_totals[lab]["PD"]);
printf(" %3d |",lab_gpu_totals[lab]["CG"]);
# Colorized:
# printf(" %s |",colour_int(lab_gpu_totals[lab]["R","normal"]));
# printf(" %s |",colour_int(lab_gpu_totals[lab]["PD","normal"]));
# printf(" %s |",colour_int(lab_gpu_totals[lab]["CG","normal"]));
#
# printf(" %s |",colour_int(lab_gpu_totals[lab]["R","overcap"]));
# printf(" %s |",colour_int(lab_gpu_totals[lab]["PD","overcap"]));
# printf(" %s |",colour_int(lab_gpu_totals[lab]["CG","overcap"]));
#
# printf(" %s |",colour_int(lab_gpu_totals[lab]["R"]));
# printf(" %s |",colour_int(lab_gpu_totals[lab]["PD"]));
# printf(" %s |",colour_int(lab_gpu_totals[lab]["CG"]));
printf("\n");
printf("| %14s |","--------------");
printf(" %3s + %3s + %3s |", "---", "---", "---");
printf(" %3s + %3s + %3s |", "---", "---", "---");
printf(" %3s + %3s + %3s |", "---", "---", "---");
printf("\n");
}
}