forked from dywsjtu/apparate
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathplot_cv_results_median.py
114 lines (95 loc) · 5.01 KB
/
plot_cv_results_median.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
# %%
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
from plotting_utils import parse_result_file
matplotlib.rcParams["figure.figsize"] = (7.5, 1.85) # (4, 1.3)
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42
# %%
models = ["ResNet18", "ResNet50", "ResNet101", "VGG11", "VGG13", "VGG16"]
datasets = ["Auburn", "Bellevue1", "Bellevue2", "Calgary", "Coral", "Hampton", "Oxford"]
# NOTE(ruipan): use these for the final version. use the abs path here for testing.
BATCH_DECISION_PATH = "../../batch_decisions/{model}_{slo_multiplier}_{arrival}.pickle"
APPARATE_LATENCY_PATH = "../../apparate_latency/{model}_{dataset}_{slo_multiplier}_{arrival}.pickle"
OPTIMAL_LATENCY_PATH = "../../optimal_latency/{model}_{dataset}_{slo_multiplier}_{arrival}_optimal.pickle"
# BATCH_DECISION_PATH = "/home/ruipan/apparate-ae/data/batch_decisions/{model}_{slo_multiplier}_{arrival}.pickle"
# APPARATE_LATENCY_PATH = "/home/ruipan/apparate-ae/data/apparate_latency/{model}_{dataset}_{slo_multiplier}_{arrival}.pickle"
# OPTIMAL_LATENCY_PATH = "/home/ruipan/apparate-ae/data/optimal_latency/{model}_{dataset}_{slo_multiplier}_{arrival}_optimal.pickle"
bar_width = 0.3 # the width of the bars
x_pos = np.arange(len(models))
patterns = ["", "\\", "/"]
colors = ["#d9ffbf", "#4f963c"]
latency_dict = {
"Apparate": [], # median runtime across all workloads for ResNet18, ResNet50, ...
"Optimal": [],
}
latency_minmax_dict = {
"Apparate": [],
"Optimal": [],
}
label_names = ["Apparate", "Optimal"]
multiplier = 0
# %%
# fill in data dict
for model in models:
all_apparate_serving_improvement = []
all_optimal_serving_improvement = []
for dataset in datasets:
print(f"parsing", model, dataset)
results = parse_result_file(
model.lower(),
dataset.lower() + "_video",
slo_multiplier=1,
arrival="fixed_30",
BATCH_DECISION_PATH=BATCH_DECISION_PATH,
APPARATE_LATENCY_PATH=APPARATE_LATENCY_PATH,
OPTIMAL_LATENCY_PATH=OPTIMAL_LATENCY_PATH,
)
apparate_serving_improvement = results["apparate_serving_improvement"]
optimal_serving_improvement = results["optimal_serving_improvement"]
serving_time_vanilla = results["serving_time_vanilla"]
serving_time_ee = results["serving_time_ee"]
serving_time_optimal = results["serving_time_optimal"]
if np.median(apparate_serving_improvement) < 0:
print(f"model {model} dataset {dataset} median improvement is negative! {np.median(apparate_serving_improvement)}")
all_apparate_serving_improvement.append(apparate_serving_improvement)
all_optimal_serving_improvement.append(optimal_serving_improvement)
print(f"within {1 - apparate_serving_improvement / optimal_serving_improvement}")
latency_dict["Apparate"].append(np.median(all_apparate_serving_improvement))
latency_dict["Optimal"].append(np.median(all_optimal_serving_improvement))
latency_minmax_dict["Apparate"].append([
max(np.median(all_apparate_serving_improvement) - min(all_apparate_serving_improvement), 2),
max(max(all_apparate_serving_improvement) - np.median(all_apparate_serving_improvement), 2),
])
latency_minmax_dict["Optimal"].append([
np.median(all_optimal_serving_improvement) - min(all_optimal_serving_improvement),
max(all_optimal_serving_improvement) - np.median(all_optimal_serving_improvement),
])
print(f"model {model}, all_apparate_serving_improvement {all_apparate_serving_improvement}")
print(f"model {model}, all_apparate_serving_improvement median {np.median(all_apparate_serving_improvement)}, min {min(all_apparate_serving_improvement)}, max {max(all_apparate_serving_improvement)}")
# %%
fig, ax = plt.subplots()
for scheme_idx, (scheme_name, median_latencies) in enumerate(latency_dict.items()):
offset = bar_width * multiplier
print(f"scheme {scheme_idx}, median_latencies {median_latencies}, median {np.median(median_latencies)}")
yerr = list(map(list, zip(*latency_minmax_dict[scheme_name]))) # transpose the list to be passed in yerr
print(f"yerr: {yerr}")
rects = ax.bar(
x_pos + offset, [round(x, 2) for x in median_latencies],
bar_width,
# https://stackoverflow.com/a/33857966: yerr values are relative to the data...
yerr=yerr,
error_kw=dict(ecolor='#666666', capsize=3,),
color=colors[scheme_idx], edgecolor='#666666',
label=scheme_name, hatch=patterns[scheme_idx],
alpha=.99,
)
# ax.bar_label(rects, padding=2, fontsize=9, color="#666666")
multiplier += 1
ax.set_ylabel('Med. Latency Wins\nvs. Vanilla (%)', fontsize=13)
ax.set_xticks(x_pos + bar_width * 0.5, models, fontsize=13)
ax.legend(loc='upper center', bbox_to_anchor=(0.5, 1.3), ncols=2, fontsize=10)
ax.set_axisbelow(True) # puts the grid below the bars
ax.grid(color='lightgrey', linestyle='dashed', axis="both", linewidth=0.8)
fig.savefig(f'cv_results_median.pdf', bbox_inches='tight', dpi=500)