data-manage: MergeRunTimeStatsToTable & calculate table statistics (#42)

* data-manage: MergeRunTimeStatsToTable & calculate table statistics

Signed-off-by: Xiaolong Fu <[email protected]>

* fmt fix

Signed-off-by: Xiaolong Fu <[email protected]>

---------

Signed-off-by: Xiaolong Fu <[email protected]>

Author: 191220029

collector/src/compile_time/binary_size/plot/plotter_cmp.py

@@ -22,4 +22,10 @@
     plt.legend(loc='lower right', prop={'size': 4})
 
     plt.tight_layout()
-    plt.savefig(out_file)
+    plt.savefig(out_file)
+
+    name_value =[item for item in zip(names, values)]
+    name_value.sort(key=lambda x: x[1])
+
+    for (name, value) in name_value:
+        print(name, ":", round(value, 2))

collector/src/lib.rs

@@ -1,7 +1,7 @@
 pub mod benchmark;
 mod compile_time;
 mod csv_transfer;
-mod execute;
+pub mod execute;
 pub mod mir_analyze;
 mod morpheme_miner;
 mod pca_analysis;

collector/src/statistics/compile_time_stat.rs

@@ -10,11 +10,11 @@ use crate::{
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct CompileTimeResult {
-    benchmark: String,
-    iteration: usize,
-    profile: Profile,
-    scenario: Scenario,
-    stats: Stats,
+    pub benchmark: String,
+    pub iteration: usize,
+    pub profile: Profile,
+    pub scenario: Scenario,
+    pub stats: Stats,
 }
 
 impl CompileTimeResult {
@@ -35,11 +35,11 @@ impl CompileTimeResult {
     }
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct CompileTimeBenchResult {
-    benchmark: String,
-    iterations: usize,
-    result_vec: Vec<CompileTimeResult>,
+    pub benchmark: String,
+    pub iterations: usize,
+    pub result_vec: Vec<CompileTimeResult>,
 }
 
 impl CompileTimeBenchResult {
@@ -90,10 +90,10 @@ impl CompileTimeBenchResult {
     }
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Clone)]
 pub struct CompileTimeResultSet {
-    id: String,
-    results: Vec<CompileTimeBenchResult>,
+    pub id: String,
+    pub results: Vec<CompileTimeBenchResult>,
 }
 
 impl CompileTimeResultSet {
@@ -141,7 +141,7 @@ impl CompileTimeResultSet {
     }
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Clone)]
 pub struct CompileTimeStatistic {
     pub name: String,
     pub profile: Profile,

data_manage/src/commannds.rs

@@ -26,9 +26,9 @@ pub enum Commands {
         #[clap(long = "rust-ver")]
         rustc: String,
 
-        /// The path of output file
+        /// The path of output directory
         #[clap(long = "out-path", default_value = "results")]
-        out_path: PathBuf,
+        out_dir: PathBuf,
     },
 
     /// Merge runtime data of the same rustc version from different benchmark groups
@@ -42,6 +42,25 @@ pub enum Commands {
         #[clap(long = "rust-ver")]
         rustc: String,
 
+        /// The path of output dir
+        #[clap(long = "out-path", default_value = "results")]
+        out_dir: PathBuf,
+    },
+
+    /// Compare 2 different datas on one metric and plot boxplot of their change rate.
+    DataCompare {
+        /// The first data input file.
+        #[clap(long = "data-1")]
+        data_a: PathBuf,
+
+        /// The second data input file.
+        #[clap(long = "data-2")]
+        data_b: PathBuf,
+
+        /// The metric that needs comparison.
+        #[clap(long = "metric")]
+        metric: String,
+
         /// The path of output file
         #[clap(long = "out-path", default_value = "results")]
         out_path: PathBuf,
@@ -105,6 +124,16 @@ pub enum Commands {
         merged_metric: String,
     },
 
+    /// Calculate statistics of a table-data fmt file.
+    CalculateTableStats {
+        /// The path of table data fmt file.
+        #[clap(long = "table-data")]
+        table_data_path: PathBuf,
+        /// The path of output file.
+        #[clap(long = "out-path")]
+        out_path: PathBuf,
+    },
+
     /// Merge compile-time stats into a table data fmt file.
     MergeCompileTimeStatsToTable {
         /// The path of table data fmt file.
@@ -121,6 +150,21 @@ pub enum Commands {
         new_metrics: String,
     },
 
+    /// Merge runtime stats into a table data fmt file.
+    MergeRuntimeStatsToTable {
+        /// The path of table data fmt file.
+        #[clap(long = "table-data")]
+        table_data_path: PathBuf,
+        /// The path of runtime stats fmt file.
+        #[clap(long = "stats")]
+        stats_path: PathBuf,
+        /// The path of output file.
+        #[clap(long = "out-path")]
+        out_path: PathBuf,
+        /// Metrics merged from stats fmt file. Use ',' to concanate the metrics.
+        #[clap(long = "new_metrics")]
+        new_metrics: String,
+    },
     /// Do pca analysis on a table fmt file.
     PcaAnalysis {
         /// The path of table data fmt file.
@@ -133,4 +177,15 @@ pub enum Commands {
         #[clap(long = "max-component-num")]
         max_component_num: u32,
     },
+
+    /// Normalize statistic by wall-time.
+    NormalizeStat {
+        /// The stat fmt file.
+        #[clap(long = "stats")]
+        stats: PathBuf,
+
+        /// The output path.
+        #[clap(long = "out-path")]
+        out_path: PathBuf,
+    },
 }

data_manage/src/compare_stats/compare_data.rs

@@ -0,0 +1,84 @@
+use std::{
+    path::PathBuf,
+    process::{Command, Stdio},
+};
+
+use super::data::{calculate_change_rate, read_data, ChangeRate};
+
+pub fn compare_data(
+    data_a: &PathBuf,
+    data_b: &PathBuf,
+    metric: &String,
+    out_path: PathBuf,
+) -> anyhow::Result<PathBuf> {
+    let data_a = read_data(data_a, metric)?;
+    let data_b = read_data(data_b, metric)?;
+
+    // Calculate change rate of stats_a on stats_b
+    let change_rate = calculate_change_rate(&data_a, &data_b);
+
+    plot_boxplot_compare(&change_rate, out_path, metric)
+}
+
+fn plot_boxplot_compare(
+    data: &ChangeRate,
+    out_path: PathBuf,
+    metric: &String,
+) -> anyhow::Result<PathBuf> {
+    let mut cmd = Command::new("python");
+    cmd.arg("src/plotters/plotter_cmp_boxplot.py")
+        .arg(
+            data.into_iter()
+                .map(|(k, v)| {
+                    format!(
+                        "{}:{}",
+                        k,
+                        v.into_iter()
+                            .map(|d| d.to_string())
+                            .collect::<Vec<String>>()
+                            .join(",")
+                    )
+                })
+                .collect::<Vec<String>>()
+                .join(";"),
+        )
+        .arg(&out_path)
+        .arg(metric);
+    cmd.stdout(Stdio::inherit());
+    cmd.spawn().unwrap().wait().unwrap();
+
+    Ok(out_path)
+}
+
+#[cfg(test)]
+mod test_compare_stat {
+    use std::{
+        fs::{self, remove_file},
+        path::PathBuf,
+    };
+
+    use super::compare_data;
+
+    /// test for compare_stat
+    ///
+    /// Step1. compare stats of metric `instructions` in `test/compare_stat/stat`.
+    ///
+    /// Step2. plot and check the compare result.
+    ///
+    /// Step3. clean up.
+    #[test]
+    fn test_compare_data() {
+        let stat_1 = PathBuf::from("test/compare_data/stat/merged-runtime_data_current.json");
+        let stat_2 = PathBuf::from("test/compare_data/stat/merged-runtime_data_old.json");
+        let metric = String::from("wall-time");
+        let out_path = PathBuf::from("test/compare_data/compare_data.jpeg");
+
+        assert_eq!(
+            out_path.clone(),
+            compare_data(&stat_1, &stat_2, &metric, out_path.clone()).unwrap()
+        );
+
+        fs::metadata(&out_path).unwrap();
+        remove_file(out_path).unwrap();
+    }
+}

data_manage/src/compare_stats/data.rs

@@ -0,0 +1,86 @@
+use std::{collections::HashMap, fs::File, io::BufReader, path::PathBuf};
+
+use anyhow::bail;
+use collector::statistics::{
+    compile_time_stat::CompileTimeResultSet, runtime_stat::RuntimeResultVec,
+};
+
+pub type LabeledData = HashMap<String, Vec<f64>>;
+pub type ChangeRate = HashMap<String, Vec<f64>>;
+
+pub fn read_data(data_file: &PathBuf, metric: &String) -> anyhow::Result<LabeledData> {
+    // Filter out non-relevant metrics and reshape stats into Hashmap
+
+    match serde_json::from_reader::<_, CompileTimeResultSet>(BufReader::new(File::open(data_file)?))
+    {
+        Ok(s) => return Ok(reshape_data(s, metric)),
+        Err(_) => (),
+    }
+
+    match serde_json::from_reader::<_, RuntimeResultVec>(BufReader::new(File::open(data_file)?)) {
+        Ok(s) => return Ok(reshape_runtime_data(s, metric)),
+        Err(e) => bail!(e),
+    }
+}
+
+pub fn calculate_change_rate(data_a: &LabeledData, data_b: &LabeledData) -> ChangeRate {
+    data_a
+        .into_iter()
+        .map(|(k, u)| {
+            (k.clone(), {
+                let mut v = data_b.get(k).unwrap().clone();
+                v.sort_by(|a, b| a.partial_cmp(b).unwrap());
+                let mut u = u.clone();
+                u.sort_by(|a, b| a.partial_cmp(b).unwrap());
+
+                let mut v = v.into_iter();
+                assert_eq!(u.len(), v.len());
+
+                u.into_iter()
+                    .map(|a| {
+                        let b = v.next().unwrap();
+                        (a - b) / b * 100.
+                    })
+                    .collect()
+            })
+        })
+        .collect()
+}
+
+fn reshape_data(data: CompileTimeResultSet, metric: &String) -> LabeledData {
+    data.results
+        .into_iter()
+        .map(|s| {
+            (
+                s.benchmark,
+                s.result_vec
+                    .into_iter()
+                    .map(|d| {
+                        d.stats
+                            .stats
+                            .into_iter()
+                            .find(|(m, _)| m == metric)
+                            .unwrap()
+                    })
+                    .map(|x| x.1)
+                    .collect(),
+            )
+        })
+        .collect()
+}
+
+fn reshape_runtime_data(data: RuntimeResultVec, metric: &String) -> LabeledData {
+    data.0
+        .into_iter()
+        .map(|s| {
+            (
+                s.name,
+                s.stats
+                    .into_iter()
+                    .map(|stats| stats.stats.into_iter().find(|(m, _)| m == metric).unwrap())
+                    .map(|x| x.1)
+                    .collect(),
+            )
+        })
+        .collect()
+}

data_manage/src/compare_stats/mod.rs

@@ -1,3 +1,5 @@
+pub mod compare_data;
 pub mod compare_stat;
 pub mod compare_stat_2d;
+mod data;
 mod stats;