data-manage: impl MergeTableMetrics & MergeCompileTimeStatsToTableBase; Mir-analyze: Fix div 0 error in oop_pattern (#40)

* Mir-analyze: reorgnize mir generate

Signed-off-by: 191220029 <[email protected]>

MirGenerate: take cargo opts when needed

Signed-off-by: 191220029 <[email protected]>

impl mir_analyze

Signed-off-by: 191220029 <[email protected]>

* Mir-analyze: Fix div 0 error in oop_pattern

Signed-off-by: 191220029 <[email protected]>

* data-manage: impl MergeTableMetrics & MergeCompileTimeStatsToTable

Signed-off-by: 191220029 <[email protected]>

---------

Signed-off-by: 191220029 <[email protected]>
Co-authored-by: 191220029 <[email protected]>

Author: 191220029

collector/src/lib.rs

@@ -2,7 +2,7 @@ pub mod benchmark;
 mod compile_time;
 mod csv_transfer;
 mod execute;
-mod mir_analyze;
+pub mod mir_analyze;
 mod morpheme_miner;
 mod pca_analysis;
 mod perf_analyze;

collector/src/mir_analyze/data/tex_writer.rs

@@ -1,14 +1,11 @@
 use std::{
-    collections::HashMap,
     fmt::Display,
     fs::File,
     io::{BufWriter, Write},
     path::Path,
 };
 
-// type TableData<U, V> = HashMap<U, Vec<V>>;
-
-pub type TableDatas<X, Y, T> = HashMap<X, HashMap<Y, T>>;
+use super::table_data::TableDatas;
 
 /// Transform 2-D data into a 2-D tex table.
 pub fn write_tex_table<X: Display + Ord, Y: Display + Ord, T: Display + Default>(

collector/src/mir_analyze/mir/oop_pattern.rs

@@ -187,6 +187,10 @@ pub fn dfc(mir_file: &MIRs) -> i32 {
     let depths_count = depths.len();
     let depths_sum: usize = depths.values().sum();
 
+    if funcname_call_count == 0 {
+        return 0;
+    }
+
     let result = funcname_call_count - depths_count + depths_sum / funcname_call_count;
     // println!("Result: {}", result);
     result as i32

collector/src/mir_analyze/mod.rs

@@ -1,4 +1,4 @@
-pub(crate) mod data;
+pub mod data;
 mod mir;
 pub(crate) mod mir_analyze;
 pub(crate) mod mir_generate;

data_manage/src/commannds.rs

@@ -88,4 +88,36 @@ pub enum Commands {
         #[clap(long = "out-path", default_value = "results")]
         out_path: PathBuf,
     },
+
+    /// Merge several metrics into a new metric of a table data fmt file.
+    MergeTableMetrics {
+        /// The path of table data fmt file.
+        #[clap(long = "table-data")]
+        table_data_path: PathBuf,
+        /// The path of output file.
+        #[clap(long = "out-path")]
+        out_path: PathBuf,
+        /// Metrics need to be merged.
+        #[clap(long = "old-metrics")]
+        old_metrics: Vec<String>,
+        /// New metric merged from old-metrics.
+        #[clap(long = "merged-metric")]
+        merged_metric: String,
+    },
+
+    /// Merge compile-time stats into a table data fmt file.
+    MergeCompileTimeStatsToTable {
+        /// The path of table data fmt file.
+        #[clap(long = "table-data")]
+        table_data_path: PathBuf,
+        /// The path of compile-time stats fmt file.
+        #[clap(long = "stats")]
+        stats_path: PathBuf,
+        /// The path of output file.
+        #[clap(long = "out-path")]
+        out_path: PathBuf,
+        /// Metrics merged from stats fmt file.
+        #[clap(long = "new_metrics")]
+        new_metrics: Vec<String>,
+    },
 }

data_manage/src/main.rs

@@ -2,10 +2,14 @@ use clap::Parser;
 use commannds::Cli;
 use compare_stats::{compare_stat::compare_stat, compare_stat_2d::compare_stat_2d};
 use merge_stats::{merge_runtime_stat::merge_runtime_stats, merge_stat::merge_compile_time_stats};
+use table_data::merge_metrics::{
+    merge_metrics_from_compile_time_stats_to_table_data, merge_metrics_on_table_data,
+};
 
 mod commannds;
 mod compare_stats;
 mod merge_stats;
+mod table_data;
 
 fn main() {
     let cli = Cli::parse();
@@ -47,5 +51,33 @@ fn main() {
             Ok(p) => println!("Write merged stats to {}", p.to_str().unwrap()),
             Err(e) => eprintln!("{}", e),
         },
+        commannds::Commands::MergeTableMetrics {
+            table_data_path,
+            out_path,
+            old_metrics,
+            merged_metric,
+        } => match merge_metrics_on_table_data(
+            &table_data_path,
+            &out_path,
+            &old_metrics,
+            &merged_metric,
+        ) {
+            Ok(p) => println!("Write merged table data to {}", p.to_str().unwrap()),
+            Err(e) => eprintln!("{}", e),
+        },
+        commannds::Commands::MergeCompileTimeStatsToTable {
+            table_data_path,
+            stats_path,
+            out_path,
+            new_metrics,
+        } => match merge_metrics_from_compile_time_stats_to_table_data(
+            &table_data_path,
+            &stats_path,
+            out_path.as_path(),
+            new_metrics,
+        ) {
+            Ok(p) => println!("Write merged table data to {}", p.to_str().unwrap()),
+            Err(e) => eprintln!("{}", e),
+        },
     }
 }

data_manage/src/table_data/merge_metrics.rs

@@ -0,0 +1,181 @@
+use std::{
+    collections::HashMap,
+    fs::File,
+    io::{BufReader, BufWriter},
+    path::{Path, PathBuf},
+};
+
+use collector::{
+    mir_analyze::data::table_data::TableDatas, statistics::compile_time_stat::CompileTimeStatistics,
+};
+
+pub fn merge_metrics_on_table_data(
+    table_data_path: &Path,
+    out_path: &Path,
+    old_metrics: &Vec<String>,
+    merged_metric: &String,
+) -> anyhow::Result<PathBuf> {
+    serde_json::to_writer(
+        BufWriter::new(File::create(out_path)?),
+        &merge_metrics(
+            serde_json::from_reader(BufReader::new(File::open(table_data_path)?))?,
+            old_metrics,
+            merged_metric,
+        ),
+    )?;
+
+    Ok(out_path.to_path_buf())
+}
+
+pub fn merge_metrics_from_compile_time_stats_to_table_data(
+    table_data_path: &Path,
+    stats_path: &Path,
+    out_path: &Path,
+    new_metrics: Vec<String>,
+) -> anyhow::Result<PathBuf> {
+    serde_json::to_writer(
+        BufWriter::new(File::create(out_path)?),
+        &merge_metrics_from_compile_time_stats(
+            serde_json::from_reader(BufReader::new(File::open(table_data_path)?))?,
+            serde_json::from_reader(BufReader::new(File::open(stats_path)?))?,
+            new_metrics,
+        ),
+    )?;
+
+    Ok(out_path.to_path_buf())
+}
+
+fn merge_metrics(
+    data: TableDatas<String, String, f64>,
+    old_metrics: &Vec<String>,
+    merged_metric: &String,
+) -> TableDatas<String, String, f64> {
+    let mut merged_metrics = TableDatas::new();
+
+    data.into_iter().for_each(|(m, d)| {
+        if old_metrics.contains(&m) {
+            match merged_metrics.get_mut(merged_metric) {
+                Some(v) => {
+                    v.iter_mut().for_each(|(k, v)| {
+                        *v += d.get(k).unwrap();
+                    });
+                }
+                None => {
+                    merged_metrics.insert(merged_metric.clone(), d);
+                }
+            }
+        } else {
+            merged_metrics.insert(m, d);
+        }
+    });
+    merged_metrics
+}
+
+fn merge_metrics_from_compile_time_stats(
+    mut data: TableDatas<String, String, f64>,
+    stats: CompileTimeStatistics,
+    new_metrics: Vec<String>,
+) -> TableDatas<String, String, f64> {
+    let stats: HashMap<String, HashMap<String, _>> = stats
+        .into_iter()
+        .map(|s| {
+            (
+                s.name,
+                s.statistic_vec
+                    .into_iter()
+                    .map(|(name, stats)| (name, stats))
+                    .collect(),
+            )
+        })
+        .collect();
+
+    new_metrics.into_iter().for_each(|m| {
+        data.insert(
+            m.clone(),
+            stats
+                .iter()
+                .map(|(b, stats_map)| (b.clone(), stats_map.get(&m).unwrap().geometric_mean))
+                .collect(),
+        );
+    });
+
+    data
+}
+
+#[cfg(test)]
+mod test_merge_metrics {
+    use std::{
+        fs::{remove_file, File},
+        io::BufReader,
+        path::PathBuf,
+    };
+
+    use collector::mir_analyze::data::table_data::TableDatas;
+
+    use crate::table_data::merge_metrics::merge_metrics_from_compile_time_stats_to_table_data;
+
+    use super::merge_metrics_on_table_data;
+
+    #[test]
+    fn test_merge_metrics_on_table_data() {
+        let table_data_path = PathBuf::from("test/table_data/data/mir-analysis.json");
+        let out_data_path = PathBuf::from("test/table_data/merged_metrics.json");
+        let old_metrics = vec![
+            String::from("oop_noc"),
+            String::from("oop_pbf"),
+            String::from("oop_dfc"),
+        ];
+        let merged_metric = String::from("oop_pattern");
+
+        assert_eq!(
+            merge_metrics_on_table_data(
+                table_data_path.as_path(),
+                out_data_path.as_path(),
+                &old_metrics,
+                &merged_metric
+            )
+            .unwrap(),
+            out_data_path
+        );
+
+        let merged_data: TableDatas<String, String, f64> =
+            serde_json::from_reader(BufReader::new(File::open(&out_data_path).unwrap())).unwrap();
+        assert!(merged_data.contains_key(&merged_metric));
+        old_metrics.iter().for_each(|m| {
+            assert!(!merged_data.contains_key(m));
+        });
+
+        remove_file(out_data_path).unwrap();
+    }
+
+    #[test]
+    fn test_merge_metrics_from_compile_time_stats_to_table_data() {
+        let table_data_path = PathBuf::from("test/table_data/data/mir-analysis.json");
+        let stats_path = PathBuf::from("test/table_data/data/merged_statstics_current.json");
+        let out_data_path = PathBuf::from("test/table_data/merged_stats_metrics.json");
+        let new_metrics = vec![
+            String::from("instructions:u"),
+            String::from("branch-misses"),
+            String::from("cache-misses"),
+        ];
+
+        assert_eq!(
+            merge_metrics_from_compile_time_stats_to_table_data(
+                table_data_path.as_path(),
+                stats_path.as_path(),
+                out_data_path.as_path(),
+                new_metrics.clone()
+            )
+            .unwrap(),
+            out_data_path
+        );
+
+        let merged_data: TableDatas<String, String, f64> =
+            serde_json::from_reader(BufReader::new(File::open(&out_data_path).unwrap())).unwrap();
+        new_metrics.iter().for_each(|m| {
+            assert!(merged_data.contains_key(m));
+        });
+
+        remove_file(out_data_path).unwrap();
+    }
+}

data_manage/src/table_data/mod.rs

@@ -0,0 +1 @@
+pub mod merge_metrics;