Merge pull request #25 from 191220029/record-compiled-binary-size

BinaryPlot: Plot change rate

collector/src/compile_time/binary_size/compare.rs

@@ -0,0 +1,68 @@
+use std::{collections::HashMap, fs::File, io::BufReader, path::PathBuf};
+
+use crate::{benchmark::profile::Profile, compile_time::result::CompileTimeBenchResult};
+
+use super::BINARY_SIZE_LABEL;
+
+/// Compare changes of binary sizes between 2 input files.
+/// Step1. Read in 2 input files.
+/// Step2. Calculate the change rate.
+pub fn compare_binary_size(
+    binary_size_a: &PathBuf,
+    binary_size_b: &PathBuf,
+    profile: Profile,
+) -> HashMap<String, f64> {
+    let read_binary_size = |p: &PathBuf| {
+        let data: Vec<CompileTimeBenchResult> =
+            serde_json::from_reader(BufReader::new(File::open(p).unwrap())).unwrap();
+        data.into_iter()
+            .map(move |i| {
+                (
+                    i.get_benchmark(),
+                    *i.get_stats_ref_by_profile(&profile)
+                        .first()
+                        .unwrap()
+                        .stats
+                        .get(BINARY_SIZE_LABEL)
+                        .unwrap(),
+                )
+            })
+            .collect::<HashMap<_, _>>()
+    };
+
+    let binary_size_a = read_binary_size(binary_size_a);
+    let binary_size_b = read_binary_size(binary_size_b);
+
+    // Calculate differences of 2 inputs.
+    let difference: HashMap<String, f64> = binary_size_a
+        .clone()
+        .iter()
+        .map(|(b, lhs)| (b.clone(), lhs - binary_size_b.get(b).unwrap()))
+        .collect();
+
+    // Calculate change ratio.
+    difference
+        .iter()
+        .map(|(b, d)| (b.clone(), { (d / binary_size_a.get(b).unwrap()) * 100. }))
+        .collect()
+}
+
+#[cfg(test)]
+mod test_compare {
+    use std::path::PathBuf;
+
+    use crate::benchmark::profile::Profile;
+
+    use super::compare_binary_size;
+
+    #[test]
+    fn test_compare_binary_size() {
+        let binary_size_a = PathBuf::from("test/binary_size/compare/binary_size_1.json");
+        let binary_size_b = PathBuf::from("test/binary_size/compare/binary_size_2.json");
+
+        assert_ne!(
+            0,
+            compare_binary_size(&binary_size_a, &binary_size_b, Profile::Release).len()
+        );
+    }
+}

collector/src/compile_time/binary_size/mod.rs

@@ -25,6 +25,7 @@ use super::result::CompileTimeBenchResult;
 
 pub mod binary_package_process;
 pub mod binary_single_process;
+pub mod compare;
 pub mod plotter;
 
 const BINARY_SIZE_LABEL: &str = "binary_size";

collector/src/compile_time/binary_size/plotter.py → collector/src/compile_time/binary_size/plot/plotter.py

@@ -46,23 +46,30 @@ def annotate_interval(y_axis: list[float], scale: int) -> list[list[float, float
     data_pair_1.sort(key=lambda d: d[1])
     data_pair_2.sort(key=lambda d: d[1])
 
-    data_1 = [np.log(item[1]) for item in data_pair_1]
-    data_2 = [np.log(item[1]) for item in data_pair_2]
-    interval_1 = annotate_interval(data_1, scale)
-    interval_2 = annotate_interval(data_2, scale)
+    data_1 = [item[1] for item in data_pair_1]
+    data_2 = [item[1] for item in data_pair_2]
+
+    geo_mean_1 = np.prod(data_1) ** (1 / len(data_1))
+    geo_mean_2 = np.prod(data_2) ** (1 / len(data_2))
+    print(f"geomean: {geo_mean_1} | {geo_mean_2}")
+
+    log_data_1 = [np.log(item) for item in data_1]
+    log_data_2 = [np.log(item) for item in data_2]
+    interval_1 = annotate_interval(log_data_1, scale)
+    interval_2 = annotate_interval(log_data_2, scale)
     annotate_1 = [item[0] for item in data_pair_1]
     annotate_2 = [item[0] for item in data_pair_2]
 
     plt.figure(dpi=500,  figsize=[scale, scale])
     plt.xticks([1, 2], [label_1, label_2])
-    plt.violinplot([data_1, data_2], showmeans=False, showmedians=True)
+    plt.violinplot([log_data_1, log_data_2], showmeans=False, showmedians=True)
 
-    plt.scatter([1]*len(data_1), data_1, color='green', marker='o', s=2)
-    plt.scatter([2]*len(data_2), data_2, color='green', marker='o', s=2)
+    plt.scatter([1]*len(log_data_1), log_data_1, color='green', marker='o', s=2)
+    plt.scatter([2]*len(log_data_2), log_data_2, color='green', marker='o', s=2)
 
-    for i, d1 in enumerate(data_1):
+    for i, d1 in enumerate(log_data_1):
         plt.annotate(annotate_1[i], (1, d1), textcoords="offset points", xytext=(interval_1[i][0], interval_1[i][1]), arrowprops=dict(headlength = 0.1, width = 0.15, headwidth = 0.1, shrink=0.99, linewidth=0.2, mutation_scale=0.1), fontsize=9)
-    for i, d2 in enumerate(data_2):
+    for i, d2 in enumerate(log_data_2):
         plt.annotate(annotate_2[i], (2, d2), textcoords="offset points", xytext=(interval_2[i][0], interval_2[i][1]), arrowprops=dict(headlength = 0.1, width = 0.15, headwidth = 0.1, shrink=0.99, linewidth=0.2, mutation_scale=0.1), fontsize=9)
 
     plt.ylabel('log (Binary Size (MB))')

collector/src/compile_time/binary_size/plot/plotter_cmp.py

@@ -0,0 +1,22 @@
+import matplotlib.pyplot as plt
+import sys
+
+if __name__ == '__main__':
+    args = sys.argv
+    assert(len(args) > 2)
+
+    raw_data = args[1].split(";")
+    out_file = args[2]
+
+    names = [item.split(',')[0] for item in raw_data]
+    values = [float(item.split(',')[1]) for item in raw_data]
+    colors = ['grey' if value >= 0 else '#A7E6BF' for value in values]
+
+    plt.figure(dpi=800, figsize=(4, 3))
+    plt.bar(names, values, color=colors)
+    plt.ylabel('Change rate of binary size (%)', fontsize=7)
+    plt.yticks(fontsize=7)
+    plt.xticks(rotation=90, fontsize=7)
+
+    plt.tight_layout()
+    plt.savefig(out_file)

collector/src/compile_time/binary_size/plotter.rs

@@ -1,4 +1,5 @@
 use std::{
+    collections::HashMap,
     fs::File,
     io::BufReader,
     path::PathBuf,
@@ -46,7 +47,7 @@ pub fn plot(
     let benchmark_binary_size_b = get_benchmark_binary_size(&data_b);
 
     let mut cmd = Command::new("python");
-    cmd.arg("src/compile_time/binary_size/plotter.py")
+    cmd.arg("src/compile_time/binary_size/plot/plotter.py")
         .arg(benchmark_binary_size_a)
         .arg(benchmark_binary_size_b)
         .arg(label_a)
@@ -58,16 +59,34 @@ pub fn plot(
     Ok(())
 }
 
+pub fn plot_compare(data: &HashMap<String, f64>, out_path: PathBuf) -> anyhow::Result<PathBuf> {
+    let mut cmd = Command::new("python");
+    cmd.arg("src/compile_time/binary_size/plot/plotter_cmp.py")
+        .arg(
+            data.into_iter()
+                .map(|(k, v)| format!("{},{}", k, v))
+                .collect::<Vec<String>>()
+                .join(";"),
+        )
+        .arg(&out_path);
+    cmd.stdout(Stdio::inherit());
+    cmd.spawn().unwrap().wait().unwrap();
+
+    Ok(out_path)
+}
+
 #[cfg(test)]
 mod test_binary_size_plotter {
     use std::{
         fs::{self, remove_file},
         path::PathBuf,
     };
 
-    use crate::benchmark::profile::Profile;
+    use crate::{
+        benchmark::profile::Profile, compile_time::binary_size::compare::compare_binary_size,
+    };
 
-    use super::plot;
+    use super::{plot, plot_compare};
 
     #[test]
     fn test_plotter() {
@@ -89,4 +108,20 @@ mod test_binary_size_plotter {
         fs::metadata(&file_path).unwrap();
         remove_file(file_path).unwrap();
     }
+
+    #[test]
+    fn test_plotter_cmp() {
+        let binary_size_a = PathBuf::from("test/binary_size/compare/binary_size_1.json");
+        let binary_size_b = PathBuf::from("test/binary_size/compare/binary_size_2.json");
+        let out_path = PathBuf::from("test/binary_size/compare/compare.jpg");
+
+        plot_compare(
+            &compare_binary_size(&binary_size_a, &binary_size_b, Profile::Release),
+            out_path.clone(),
+        )
+        .unwrap();
+
+        fs::metadata(&out_path).unwrap();
+        remove_file(out_path).unwrap();
+    }
 }

collector/src/main.rs

@@ -8,7 +8,13 @@ use std::{
 use anyhow::{Context, Ok};
 use benchmark::scenario::Scenario;
 use clap::Parser;
-use compile_time::{bench_compile_time, binary_size::plotter::plot};
+use compile_time::{
+    bench_compile_time,
+    binary_size::{
+        compare::compare_binary_size,
+        plotter::{plot, plot_compare},
+    },
+};
 use runtime::bench_runtime;
 use toolchain::{Cli, Commands, ResultWriter};
 
@@ -279,10 +285,17 @@ fn main_result() -> anyhow::Result<i32> {
             data2_label,
             profile,
             out_path,
-        } => {
-            plot(data1, data2, data1_label, data2_label, out_path, profile)?;
-            Ok(0)
-        }
+            mode,
+        } => match mode {
+            toolchain::BinaryPlotMode::DefaultMode => {
+                plot(data1, data2, data1_label, data2_label, out_path, profile)?;
+                Ok(0)
+            }
+            toolchain::BinaryPlotMode::CompareMode => {
+                plot_compare(&compare_binary_size(&data1, &data2, profile), out_path)?;
+                Ok(0)
+            }
+        },
     }
 }
 

collector/src/toolchain.rs

@@ -288,24 +288,27 @@ pub enum Commands {
 
     /// Boxplot and scatterplot for a pair of compiled_binary_size data for comparsion.
     BinaryPlot {
-        /// The path of compiled_binary_size data 1
+        /// The path of compiled_binary_size data 1.
         #[clap(long = "data1")]
         data1: PathBuf,
-        /// The path of compiled_binary_size data 2
+        /// The path of compiled_binary_size data 2.
         #[clap(long = "data2")]
         data2: PathBuf,
-        /// The label of compiled_binary_size data 1
-        #[clap(long = "data1-label")]
+        /// The label of compiled_binary_size data 1. Deprecated when mode is `cmp`.
+        #[clap(long = "data1-label", default_value = "")]
         data1_label: String,
-        /// The label of compiled_binary_size data 2
-        #[clap(long = "data2-label")]
+        /// The label of compiled_binary_size data 2. Deprecated when mode is `cmp`.
+        #[clap(long = "data2-label", default_value = "")]
         data2_label: String,
-        /// The profile of data collected
+        /// The profile of data collected.
         #[clap(long = "profile")]
         profile: Profile,
-        /// The path of output figure
+        /// The path of output figure.
         #[clap(long = "out-path")]
         out_path: PathBuf,
+        /// The mode of plotter. Either be `default` or be `cmp`(compare).
+        #[clap(long = "mode", default_value = "default")]
+        mode: BinaryPlotMode,
     },
 
     /// Trasfer Json outpu to csv output.
@@ -420,6 +423,23 @@ impl FromStr for PerfTool {
     }
 }
 
+#[derive(Debug)]
+pub enum BinaryPlotMode {
+    DefaultMode,
+    CompareMode,
+}
+
+impl FromStr for BinaryPlotMode {
+    type Err = String;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "default" => std::result::Result::Ok(Self::DefaultMode),
+            "cmp" => std::result::Result::Ok(Self::CompareMode),
+            _ => Err(format!("Unrecognized BinaryPlotMode {}. BinaryPlotMode should be either `default` or `cmp`", s)),
+        }
+    }
+}
+
 pub struct ResultWriter {
     fptr: File,
 }