Added jaccard index heatmap for edges

Author: ctrlaltaf

Snakefile

@@ -395,6 +395,8 @@ rule evaluation:
         pr_edge_png = SEP.join([out_dir, '{dataset_gold_standard_pairs}-eval', 'precision-recall-per-pathway_edge.png']),
         pr_curve_png = SEP.join([out_dir, '{dataset_gold_standard_pairs}-eval', 'precision-recall-curve-ensemble-nodes.png']),
         pca_chosen_pr_file = SEP.join([out_dir, '{dataset_gold_standard_pairs}-eval', "precision-recall-pca-chosen-pathway.txt"]),
+        heatmap_edge_file = SEP.join([out_dir, '{dataset_gold_standard_pairs}-eval', "jaccard_edge_heatmap.png"]),
+
     run:
         node_table = Evaluation.from_file(input.gold_standard_file).node_table
         edge_table = Evaluation.from_file(input.gold_standard_file).edge_table
@@ -404,6 +406,7 @@ rule evaluation:
         Evaluation.precision_recall_curve_node_ensemble(node_ensemble, node_table, output.pr_curve_png)
         pca_chosen_pathway = Evaluation.pca_chosen_pathway(input.pca_coordinates_file, out_dir)
         Evaluation.precision_and_recall_node(pca_chosen_pathway, node_table, algorithms, output.pca_chosen_pr_file)
+        Evaluation.jaccard_edge_heatmap(input.pathways, edge_table, algorithms, output.pr_edge_file, output.heatmap_edge_file)
 
 
 # Returns all pathways for a specific algorithm and dataset

config/config.yaml

@@ -128,7 +128,7 @@ gold_standards:
       dataset_labels: ["data0"]
     -
       label: gs1
-      node_files: ["gs_nodes1.txt"]
+      node_files: ["gs_nodes1.txt"] # why list?
       data_dir: "input"
       dataset_labels: ["data1", "data0"]
 

config/synthetic.yaml

@@ -51,15 +51,15 @@ algorithms:
 
       - name: "omicsintegrator1"
         params:
-              include: true
+              include: false
               run1:
                   b: [5, 6]
                   w: np.linspace(0,5,2)
                   d: [10]
 
       - name: "omicsintegrator2"
         params:
-              include: true
+              include: false
               run1:
                   b: [4]
                   g: [0]
@@ -77,7 +77,7 @@ algorithms:
 
       - name: "mincostflow"
         params:
-              include: true
+              include: false
               run1:
                   flow: [1] # The flow must be an int
                   capacity: [1]
@@ -108,14 +108,24 @@ datasets:
       other_files: []
       # Relative path from the spras directory
       data_dir: "input"
+    -
+      # Labels can only contain letters, numbers, or underscores
+      label: data3
+      node_files: ["node-prizes-synthetic.txt", "sources.txt", "targets.txt"]
+      # DataLoader.py can currently only load a single edge file, which is the primary network
+      edge_files: ["network-synthetic.txt"]
+      # Placeholder
+      other_files: []
+      # Relative path from the spras directory
+      data_dir: "input"
 
 gold_standards:
     -
       label: gs2
       node_files: ["gs_nodes2.txt"]
       edge_files: ["gs_edges.txt"]
       data_dir: "input"
-      dataset_labels: ["data2"]
+      dataset_labels: ["data2", "data3"]
 
 # If we want to reconstruct then we should set run to true.
 # TODO: if include is true above but run is false here, algs are not run.

spras/evaluation.py

@@ -12,6 +12,7 @@
     precision_score,
     recall_score,
 )
+import seaborn as sns
 
 
 class Evaluation:
@@ -211,6 +212,54 @@ def precision_and_recall_node(file_paths: Iterable[Path], node_table: pd.DataFra
                 plt.savefig(output_png)
 
 
+    @staticmethod
+    def jaccard_edge_heatmap(file_paths: Iterable[Path], edge_table: pd.DataFrame, algorithms: list, output_file: str, output_png:str=None):
+        """
+        Takes in file paths for a specific dataset and an associated gold standard edge table.
+        Calculates precision and recall for each pathway file
+        Returns output back to output_file
+        @param file_paths: file paths of pathway reconstruction algorithm outputs
+        @param edge_table: the gold standard edges
+        @param algorithms: list of algorithms used in current run of SPRAS
+        @param output_file: the filename to save the precision and recall of each pathway
+        @param output_png (optional): the filename to plot the precision and recall of each pathway (not a PRC)
+        """
+        print("jaccard_heatmap")
+
+        gs_edges = set()
+        for row in edge_table.itertuples():
+            gs_edges.add((row[1], row[2]))
+        # calculate all the jaccard edge index for each method against the gs
+        jaccard_edges_indices_list = []
+        algs = []
+        for file in file_paths:
+            df = pd.read_table(file, sep="\t", header=0, usecols=["Node1", "Node2"])
+            method_edges = set()
+            for row in df.itertuples():
+                method_edges.add((row[1], row[2]))
+            edge_union = gs_edges | method_edges
+            edge_intersection = gs_edges & method_edges
+            jaccard_edge_index = len(edge_intersection) / len(edge_union)
+            jaccard_edges_indices_list.append(float(jaccard_edge_index))
+            algs.append(file.split("/")[1].split("-")[1])
+
+        jaccard_edges_indices = np.asanyarray([jaccard_edges_indices_list])
+
+        print(algs)
+
+        plt.figure(figsize=(10, 8))
+        sns.heatmap(
+            jaccard_edges_indices,
+            annot=True,
+            cmap="viridis",
+            yticklabels=["Pathways"],
+            xticklabels=algs,
+        )
+        plt.xlabel("Algorithms")
+        plt.title("Jaccard Index Edge Heatmap")
+        plt.savefig(output_png, format="png", dpi=300)
+
+
     def select_max_freq_and_node(row: pd.Series):
         """
         Selects the node and frequency with the highest frequency value from two potential nodes in a row.