bug fix

Author: ppashakhanloo

codeql_setup.sh

@@ -3,7 +3,7 @@
 rm -rf codeql_dir
 mkdir codeql_dir
 cd codeql_dir
-wget https://github.com/github/codeql-cli-binaries/releases/download/v2.4.1/codeql-linux64.zip
+wget https://github.com/github/codeql-cli-binaries/releases/download/v2.5.0/codeql-linux64.zip
 unzip codeql-linux64.zip
 git clone https://github.com/github/codeql.git codeql_repo
 

data_prep/graph/build_graph.py

@@ -5,20 +5,19 @@
 
 def main(args: List[str]) -> None:
     if not len(args) == 4:
-        print('Usage: python3 build_graph.py <facts_dir> <join_filepath> <output_file>')
+        print('Usage: python3 build_graph.py <tables_dir> <join_filepath> <output_file>')
         exit(1)
 
-    facts_dir = args[1]
+    tables_dir = args[1]
     join_filepath = args[2]
     output_file = args[3]
 
     language = 'python'
-    graph_builder = GraphBuilder(facts_dir, join_filepath, language)
+    graph_builder = GraphBuilder(tables_dir, join_filepath, language)
     # When include_callgraph=True then callgraphs are added to the graph.
     # The default is include_callgraph=False.
-    graph = graph_builder.build(include_callgraph=True)
+    graph = graph_builder.build(include_callgraph=False)
     GraphBuilder.save_gv(graph, output_file + '.gv')
 
-
 if __name__ == "__main__":
     main(sys.argv)

data_prep/graph/graphbuilder.py

@@ -1,18 +1,20 @@
 import csv
+import json
 import os
 import graphviz
+import networkx as nx
 from io import StringIO
 from data_prep.random_walk.walkutils import WalkUtils, JavaWalkUtils
 from typing import List, Set, Dict, Tuple
 
 
 class GraphBuilder:
-    facts_dir = None      # type: str
+    tables_dir = None      # type: str
     join_filepath = None  # type: str
     language = None       # type: str
 
-    def __init__(self, facts_dir: str, join_filepath: str, language: str):
-        self.facts_dir = facts_dir
+    def __init__(self, tables_dir: str, join_filepath: str, language: str):
+        self.tables_dir = tables_dir
         self.join_filepath = join_filepath
         self.language = GraphBuilder.parse_language(language)
 
@@ -33,17 +35,9 @@ def load_columns(self) -> Dict[str, List[str]]:
         else:
             raise ValueError('Cannot load columns for language:', self.language)
 
-    def load_fact_table(self, fact_file: str) -> List[Tuple]:
+    def load_table(self, table_file: str) -> List[Tuple]:
         table = []
-        with open(self.facts_dir + '/' + fact_file, 'r') as f:
-            reader = csv.reader(f, delimiter=',')
-            for line in reader:
-                table.append(tuple(line))
-        return table
-
-    def load_csv_table(self, fact_file: str) -> List[Tuple]:
-        table = []
-        with open(self.facts_dir + '/' + fact_file, 'r') as f:
+        with open(self.tables_dir + '/' + table_file, 'r') as f:
             data = f.read()
             data = data.replace('\x00', '**NULLBYTE**', -1)
             reader = csv.reader(StringIO(data), delimiter=',')
@@ -88,54 +82,20 @@ def load_joins(self) -> (Dict, Dict):
 
         return joins, keys
 
-    def load_db(self, col_dict: Dict) -> Dict[str, List[Tuple]]:
+    def load_db(self, col_dict: Dict, include_callgraph=False) -> Dict[str, List[Tuple]]:
         database = {}  # type: Dict[str, List[Tuple]]
-        for fact_file in os.listdir(self.facts_dir):
-            if fact_file.endswith('.csv.facts'):
-                table_name = fact_file[:-10]  # remove ".csv.facts"
-                if table_name in col_dict:
-                    database[table_name] = self.load_fact_table(fact_file)
-            elif fact_file.endswith('.bqrs.csv'):
-                table_name = fact_file[:-9]  # remove ".bqrs.csv"
-                if table_name in col_dict:
-                    database[table_name] = self.load_csv_table(fact_file)
-            elif fact_file.endswith('.csv'):
-                table_name = fact_file[:-4]  # remove ".csv"
+        for table_file in os.listdir(self.tables_dir):
+            if table_file.endswith('.csv'):
+                if 'call_graph' in table_file and not include_callgraph:
+                    continue # skip the call_graph if call graph is not requested.
+                elif 'call_graph' in table_file and include_callgraph:
+                    table_name = 'call_graph'
+                else:
+                    table_name = table_file[:table_file.find('.')] # remove suffixes
                 if table_name in col_dict:
-                    database[table_name] = self.load_csv_table(fact_file)
+                    database[table_name] = self.load_table(table_file)
         return database
 
-    def load_callgraph(self):
-        if self.language == 'python':
-            call_graph = []
-            if 'one_hop_call_graph.csv.facts' in os.listdir(self.facts_dir):
-                callgraph_file = 'one_hop_call_graph.csv.facts'
-            elif 'full_call_graph.csv.facts' in os.listdir(self.facts_dir):
-                callgraph_file = 'full_call_graph.csv.facts'
-            elif 'one_hop_call_graph.csv' in os.listdir(self.facts_dir):
-                callgraph_file = 'one_hop_call_graph.csv'
-            elif 'full_call_graph.csv' in os.listdir(self.facts_dir):
-                callgraph_file = 'full_call_graph.csv'
-            elif 'one_hop_call_graph.bqrs.csv' in os.listdir(self.facts_dir):
-                callgraph_file = 'one_hop_call_graph.bqrs.csv'
-            elif 'full_call_graph.bqrs.csv' in os.listdir(self.facts_dir):
-                callgraph_file = 'full_call_graph.bqrs.csv'
-            else:
-                return []
-
-            with open(os.path.join(self.facts_dir, callgraph_file), 'r') as cf:
-                for line in cf.readlines():
-                    call_edge = line.split(',')
-                    call_graph.append([s.strip() for s in call_edge])
-            if callgraph_file.endswith('facts'):
-                return call_graph
-            else:
-                return call_graph [1:]
-        elif self.language == 'java':
-            raise NotImplementedError
-        else:
-            raise ValueError('Cannot load the call graph for langauge:', self.language)
-
     @staticmethod
     def build_value_to_tuple_map(db: Dict, keys: Dict) -> Dict[str, Set[Tuple[Tuple, str]]]:
         val_tuple_map = {}  # type: Dict[str, Set[Tuple[Tuple, str]]]
@@ -182,7 +142,7 @@ def labels_if_joinable(joins: Dict, columns: Dict, l_rel: str, l_tuple: Tuple, r
                 labels.append(GraphBuilder.edb_edge_label(l_rel, l_col, r_rel, r_col))
         return labels
 
-    def build_graph(self, db: Dict, joins: Dict, keys: Dict, columns: Dict, call_graph: List) -> graphviz.Graph:
+    def build_graph(self, db: Dict, joins: Dict, keys: Dict, columns: Dict) -> graphviz.Graph:
         val_tuple_map = self.build_value_to_tuple_map(db, keys)  # type: Dict[str, Set[Tuple[Tuple, str]]]
         tuple_index_map = {}  # type: Dict[str, Dict[Tuple, int]]
         index_rel_map = {}    # type: Dict[int, str]
@@ -199,6 +159,7 @@ def build_graph(self, db: Dict, joins: Dict, keys: Dict, columns: Dict, call_gra
                 index += 1
         # Build the graph
         graph = graphviz.Graph()
+
         for i in index_tuple_map:
             graph.node(str(i), self.edb_tuple_label(index_tuple_map[i], index_rel_map[i]))
         for val in val_tuple_map:
@@ -216,33 +177,13 @@ def build_graph(self, db: Dict, joins: Dict, keys: Dict, columns: Dict, call_gra
                     for edge_label in edge_labels:
                         graph.edge(str(index_i), str(index_j), edge_label)
 
-        def search_py_exprs(expr_id):
-            indices = []
-            for i in index_rel_map:
-                if index_rel_map[i] == 'py_exprs':
-                    indices.append(i)
-            for i in indices:
-                if index_tuple_map[i][0] == expr_id:
-                    return i
-            return None
-
-        for call_edge in call_graph:
-            function1_expr_id = search_py_exprs(call_edge[0])
-            function2_expr_id = search_py_exprs(call_edge[1])
-            edge_label = '(callgraph.0,call_graph.1)'
-            if function1_expr_id and function2_expr_id:
-                graph.edge(str(function1_expr_id), str(function2_expr_id), edge_label)
         return graph
 
     def build(self, include_callgraph=False) -> graphviz.Graph:
         columns = self.load_columns()
-        db = self.load_db(columns)
-        if include_callgraph:
-            callgraph = self.load_callgraph()
-        else:
-            callgraph = []
+        db = self.load_db(columns, include_callgraph)
         joins, keys = self.load_joins()
-        graph = self.build_graph(db, joins, keys, columns, callgraph)
+        graph = self.build_graph(db, joins, keys, columns)
         return graph
 
     @staticmethod

data_prep/random_walk/datapoint.py

@@ -85,4 +85,4 @@ def to_dict(self) -> Dict:
 
     def dump_json(self, filepath: str) -> None:
         with open(filepath, 'w') as outfile:
-            json.dump(self.to_dict(), outfile, indent=4)
+            json.dump(self.to_dict(), outfile)

data_prep/random_walk/randomwalk.py

@@ -3,14 +3,10 @@
 from random import choice, choices, randint
 from typing import List, Dict, Set
 from data_prep.random_walk.walkutils import WalkUtils, JavaWalkUtils
-import time
-
-def ms():
-    return round(time.time() * 1000)
 
 class RandomWalker:
     BIAS_WEIGHT = 5
-    PYTHON_BIAS_TABLES = {'py_exprs', 'py_stmts', 'variable', 'py_variables'}
+    PYTHON_BIAS_TABLES = {'py_exprs', 'py_stmts', 'py_variables'}
     JAVA_BIAS_TABLES = {'exprs', 'stmts'}
 
     language = None         # type: str
@@ -123,8 +119,9 @@ def build_node_map(graph: AGraph, language: str) -> Dict[Node, str]:
 
     @staticmethod
     def load_graph_from_gv(path: str) -> AGraph:
-        graph = AGraph(path, directed=False)
-        return nx.nx_agraph.from_agraph(graph)
+        g = AGraph()
+        g.read(path)
+        return nx.nx_agraph.from_agraph(g)
 
     # Padding the given list to size `num` by duplicating elements that are
     # selected from the list at random.

data_prep/random_walk/walkutils.py

@@ -45,7 +45,8 @@ class WalkUtils:
         'py_ssa_phi': ['phi', 'arg'],
         'py_ssa_var': ['id', 'var'],
         'py_ssa_use': ['node', 'var'],
-        'py_ssa_defn': ['id', 'node']
+        'py_ssa_defn': ['id', 'node'],
+        'call_graph': ['f1def', 'f2def', 'f1', 'f2', 'call']
     }
 
     expr_kinds = {  # type: Dict[int, str]
@@ -122,11 +123,6 @@ class WalkUtils:
     def gen_node_type_value(rel_name: str, values: List[str]):
         node_type = rel_name
         node_value = ''
-        # Compute node values
-        if rel_name == 'py_bytes':
-            node_value = values[0]
-        if rel_name == 'py_numbers':
-            node_value = values[0]
         if rel_name == 'variable':
             node_type = 'v_' + values[0]
             node_value = values[2]
@@ -138,19 +134,6 @@ def gen_node_type_value(rel_name: str, values: List[str]):
         if rel_name == 'py_stmts':
             kind = int(values[1])
             node_type = 'stmt_' + WalkUtils.stmt_kinds[kind]
-        
-        ##### uncomment the following lines for more node values #####
-        #if rel_name == 'py_boolops':
-        #    node_value = values[1]
-        #if rel_name == 'py_cmpops':
-        #    node_value = values[1]
-        #if rel_name == 'py_dict_items':
-        #    node_value = values[1]
-        #if rel_name == 'py_operators':
-        #    node_value = values[1]
-        #if rel_name == 'py_unaryops':
-        #    node_value = values[1]
-        ###############################################################
 
         # Otherwise, use relation name as the label
         return node_type, node_value

dbwalk/data_util/cook_from_gv_stub.py

@@ -73,6 +73,7 @@
                 json_name = cand + sep + '.'.join(tmp.split('.')[:-1]) + '.json'
                 if json_name in json_files:
                     break
+            # meta_data_list contains the parsed contents of *.json file.
             with open(os.path.join(folder, json_name), 'r') as f:
                 meta_data_list = json.load(f)
             for meta_data in meta_data_list:
@@ -84,15 +85,15 @@
             pbar.set_description('#n: %d, #e: %d, #v: %d, #t: %d' % (len(node_types), len(edge_types), max_num_vars, len(token_vocab)))
         if len(gh):
             gh.dump(os.path.join(out_folder, 'chunk_%d' % chunk_idx))
-            
+
     print('# node types', len(node_types))
     print('# edge types', len(edge_types))
     print('max # vars per program', max_num_vars)
     print('# tokens', len(token_vocab))
 
     var_dict = {}
     var_reverse_dict = {}
-    max_num_vars = min(max_num_vars, 100)
+    #max_num_vars = min(max_num_vars, 200)
     for i in range(max_num_vars):
         val = get_or_add(node_types, var_idx2name(i))
         var_dict[i] = val

scaled_data_gen/join.txt python_codeql/join.txtscaled_data_gen/join.txt renamed to python_codeql/join.txt

@@ -85,3 +85,4 @@ py_variables.1 <----> py_exprs.0
 variable.1 <----> py_Classes.0
 variable.1 <----> py_Functions.0
 variable.1 <----> py_Modules.0
+call_graph.0 <----> call_graph.1