Code Llama Tool Integration

.github/workflows/main.yml

@@ -38,11 +38,12 @@ jobs:
       uses: stCarolas/setup-maven@v4.1
     - name: Creating env-config.json
       run: |
-        cd /home/runner/work/SMAT/SMAT/nimrod/tests/
+        repo_name=$(basename $GITHUB_REPOSITORY)
+        cd /home/runner/work/$repo_name/$repo_name/nimrod/tests/
         java_path="/opt/hostedtoolcache/Java_Adopt_jdk/$(ls /opt/hostedtoolcache/Java_Adopt_jdk)/x64"
         contents="$(jq --arg java_path "$java_path" '.java_home=$java_path | .maven_home = "/opt/hostedtoolcache/maven/3.5.4/x64"' env-config.json)"
         echo "${contents}" > env-config.json
-        cd /home/runner/work/SMAT/SMAT
+        cd /home/runner/work/$repo_name/$repo_name
     - name: Test with pytest
       run: |
         pytest -k 'not test_general_behavior_study_semantic_conflict'

nimrod/__main__.py

@@ -16,6 +16,7 @@
 from nimrod.test_suite_generation.generators.randoop_test_suite_generator import RandoopTestSuiteGenerator
 from nimrod.test_suite_generation.generators.evosuite_differential_test_suite_generator import EvosuiteDifferentialTestSuiteGenerator
 from nimrod.test_suite_generation.generators.evosuite_test_suite_generator import EvosuiteTestSuiteGenerator
+from nimrod.test_suite_generation.generators.codellama_test_suite_generator import CodellamaTestSuiteGenerator
 from nimrod.test_suite_generation.generators.project_test_suite_generator import ProjectTestSuiteGenerator
 from nimrod.test_suites_execution.main import TestSuitesExecution, TestSuiteExecutor
 from nimrod.tools.bin import MOD_RANDOOP, RANDOOP
@@ -26,7 +27,7 @@
 
 def get_test_suite_generators(config: Dict[str, str]) -> List[TestSuiteGenerator]:
   config_generators = config.get(
-      'test_suite_generators', ['randoop', 'randoop-modified', 'evosuite', 'evosuite-differential', 'project'])
+      'test_suite_generators', ['randoop', 'randoop-modified', 'evosuite', 'evosuite-differential', 'codellama', 'project'])
   generators: List[TestSuiteGenerator] = list()
 
   if 'randoop' in config_generators:
@@ -38,6 +39,8 @@ def get_test_suite_generators(config: Dict[str, str]) -> List[TestSuiteGenerator
     generators.append(EvosuiteTestSuiteGenerator(Java()))
   if 'evosuite-differential' in config_generators:
     generators.append(EvosuiteDifferentialTestSuiteGenerator(Java()))
+  if 'codellama' in config_generators:
+    generators.append(CodellamaTestSuiteGenerator(Java()))
   if 'project' in config_generators:
     generators.append(ProjectTestSuiteGenerator(Java()))
 
@@ -95,7 +98,7 @@ def main():
     if scenario.run_analysis:
       smat.run_tool_for_semmantic_conflict_detection(scenario)
     else:
-      logging.info(f"Skipping tool execution for project f{scenario.project_name}")
+      logging.info(f"Skipping tool execution for project {scenario.project_name}")
 
 
 if __name__ == '__main__':

nimrod/core/merge_scenario_under_analysis.py

@@ -1,8 +1,8 @@
-from typing import List, Dict
+from typing import List, Dict, Union
 
 
 class MergeScenarioUnderAnalysis:
-    def __init__(self, project_name: str, run_analysis: bool, scenario_commits: "ScenarioInformation", targets: "Dict[str, List[str]]", scenario_jars: "ScenarioInformation", jar_type: str):
+    def __init__(self, project_name: str, run_analysis: bool, scenario_commits: "ScenarioInformation", targets: "Dict[str, Union[List[Dict[str, str]], List[str]]]", scenario_jars: "ScenarioInformation", jar_type: str):
         self.project_name = project_name
         self.run_analysis = run_analysis
         self.scenario_commits = scenario_commits

nimrod/output_generation/output_generator.py

@@ -11,7 +11,8 @@
 
 
 class OutputGenerator(ABC, Generic[T]):
-    REPORTS_DIRECTORY = path.join(get_base_output_path(), "reports")
+    parent_dir = path.dirname(get_base_output_path())
+    REPORTS_DIRECTORY = path.join(parent_dir, "reports")
 
     def __init__(self, report_name: str) -> None:
         super().__init__()
@@ -27,9 +28,27 @@ def write_report(self, context: OutputGeneratorContext) -> None:
         file_path = path.join(self.REPORTS_DIRECTORY, self._report_name)
 
         logging.info(f"Starting data processing of {self._report_name} report")
-        data = self._generate_report_data(context)
+        new_data = self._generate_report_data(context)
         logging.info(f"Finished data processing of {self._report_name} report")
 
-        with open(file_path, "w") as write:
-            json.dump(data, write)
+        existing_data = self._load_existing_data(file_path)
+
+        if not isinstance(existing_data, list):
+            existing_data = [existing_data] if existing_data else []
+        existing_data.append(new_data)
+
+        self._write_json(file_path, existing_data)
         logging.info(f"Finished generation of {self._report_name} report")
+
+    def _load_existing_data(self, file_path: str):
+        if not path.exists(file_path):
+            return []
+        try:
+            with open(file_path, "r") as read_file:
+                return json.load(read_file)
+        except json.JSONDecodeError:
+            return []
+
+    def _write_json(self, file_path: str, data) -> None:
+        with open(file_path, "w") as write_file:
+            json.dump(data, write_file, indent=4)

nimrod/output_generation/semantic_conflicts_output_generator.py

@@ -1,8 +1,9 @@
-from typing import Dict, List, TypedDict
+from typing import Dict, List, TypedDict, Union
 from nimrod.output_generation.output_generator import OutputGenerator, OutputGeneratorContext
 from nimrod.test_suites_execution.main import TestSuitesExecution
 from os import path
 from bs4 import BeautifulSoup
+import logging
 
 
 class SemanticConflictsOutput(TypedDict):
@@ -12,7 +13,7 @@ class SemanticConflictsOutput(TypedDict):
     test_case_name: str
     test_case_results: Dict[str, str]
     test_suite_path: str
-    scenario_targets: Dict[str, List[str]]
+    scenario_targets: Dict[str, Union[List[Dict[str, str]], List[str]]]
     exercised_targets: Dict[str, List[str]]
 
 
@@ -26,40 +27,51 @@ def _generate_report_data(self, context: OutputGeneratorContext) -> List[Semanti
 
         for semantic_conflict in context.semantic_conflicts:
             # We need to detect which targets from the input were exercised in this conflict.
-            coverage_report_root = self._test_suites_execution.execute_test_suite_with_coverage(
-                test_suite=semantic_conflict.detected_in.test_suite,
-                target_jar=context.scenario.scenario_jars.merge,
-                test_cases=[semantic_conflict.detected_in.name]
-            )
-
-            exercised_targets = self._extract_exercised_targets_from_coverage_report(
-                coverage_report_root=coverage_report_root,
-                targets=context.scenario.targets
-            )
-
-            report_data.append({
-                "project_name": context.scenario.project_name,
-                "scenario_commits": context.scenario.scenario_commits.__dict__,
-                "criteria": semantic_conflict._satisfying_criteria.__class__.__name__,
-                "test_case_name": semantic_conflict.detected_in.name,
-                "test_case_results": {
-                    "base": semantic_conflict.detected_in.base,
-                    "left": semantic_conflict.detected_in.left,
-                    "right": semantic_conflict.detected_in.right,
-                    "merge": semantic_conflict.detected_in.merge
-                },
-                "test_suite_path": semantic_conflict.detected_in.test_suite.path,
-                "scenario_targets": context.scenario.targets,
-                "exercised_targets": exercised_targets
-            })
+            exercised_targets: Dict[str, List[str]] = dict()
+            try:
+                coverage_report_root = self._test_suites_execution.execute_test_suite_with_coverage(
+                    test_suite=semantic_conflict.detected_in.test_suite,
+                    target_jar=context.scenario.scenario_jars.merge,
+                    test_cases=[semantic_conflict.detected_in.name]
+                )
+
+                exercised_targets = self._extract_exercised_targets_from_coverage_report(
+                    coverage_report_root=coverage_report_root,
+                    targets=context.scenario.targets
+                )
+
+            except Exception as e:
+                # If we cannot execute the test suite with coverage, we log the error and continue.
+                logging.error("Error executing test suite with coverage for semantic conflict: %s", e)
+
+            finally:
+                report_data.append({
+                    "project_name": context.scenario.project_name,
+                    "scenario_commits": context.scenario.scenario_commits.__dict__,
+                    "criteria": semantic_conflict._satisfying_criteria.__class__.__name__,
+                    "test_case_name": semantic_conflict.detected_in.name,
+                    "test_case_results": {
+                        "base": semantic_conflict.detected_in.base,
+                        "left": semantic_conflict.detected_in.left,
+                        "right": semantic_conflict.detected_in.right,
+                        "merge": semantic_conflict.detected_in.merge
+                    },
+                    "test_suite_path": semantic_conflict.detected_in.test_suite.path,
+                    "scenario_targets": context.scenario.targets,
+                    "exercised_targets": exercised_targets
+                })
 
         return report_data
 
-    def _extract_exercised_targets_from_coverage_report(self, coverage_report_root: str, targets: Dict[str, List[str]]):
+    def _extract_exercised_targets_from_coverage_report(self, coverage_report_root: str, targets: Dict[str, Union[List[Dict[str, str]], List[str]]]):
         exercised_targets: Dict[str, List[str]] = dict()
 
         for class_name in targets.keys():
-            for method_name in targets[class_name]:
+            for method_item in targets[class_name]:
+                if isinstance(method_item, dict):
+                    method_name = method_item.get("method", "")
+                else:
+                    method_name = method_item
                 if self._was_target_exercised(coverage_report_root, class_name, method_name):
                     exercised_targets[class_name] = exercised_targets.get(
                         class_name, []) + [method_name]
@@ -80,7 +92,11 @@ def _was_target_exercised(self, coverage_report_root: str, fqcn: str, method_sig
         # We itereate in each method row
         for method_row in method_report_rows:
             if method_row.get_text().find(method_name) != -1:
-                if method_row.select_one('td:nth-last-child(2)').get_text() == '0':
+                tag = method_row.select_one('td:nth-last-child(2)')
+                if tag is None:
+                    continue
+                # If the second last column is 0, it means the method was not executed
+                if tag.get_text() == "0":
                     return True
 
         return False

nimrod/output_generation/test_suites_output_generator.py

@@ -1,4 +1,4 @@
-from typing import List, TypedDict
+from typing import List, TypedDict, Dict, Union
 from nimrod.dynamic_analysis.behavior_change import BehaviorChange
 from nimrod.dynamic_analysis.semantic_conflict import SemanticConflict
 from nimrod.output_generation.output_generator import OutputGenerator, OutputGeneratorContext
@@ -7,6 +7,7 @@
 
 class TestSuitesOutput(TypedDict):
     project_name: str
+    targets: Dict[str, Union[List[Dict[str, str]], List[str]]]
     generator_name: str
     path: str
     detected_semantic_conflicts: bool
@@ -23,6 +24,7 @@ def _generate_report_data(self, context: OutputGeneratorContext) -> List[TestSui
         for test_suite in context.test_suites:
             report_data.append({
                 "project_name": context.scenario.project_name,
+                "targets": context.scenario.targets,
                 "generator_name": test_suite.generator_name,
                 "path": test_suite.path,
                 "detected_semantic_conflicts": self._has_detected_semantic_conflicts_in_test_suite(test_suite, context.semantic_conflicts),

nimrod/report_metrics/coverage/coverage_report.py

@@ -94,9 +94,6 @@ def get_valid_test_suite(self, toolSuites, first_entry, last_entry):
         return None
 
     def retornaDadosParaAnalise(self, evo, path_suite, suite_merge, jacoco, classeTarget, listaPacoteMetodoClasse, targets: "dict[str, list[str]]"):
-        global tagAClasseTarget
-        global tagSpanMetodoTarget
-
         print("Classe Target ", classeTarget)
 
         listaJar = evo.project_dep.mergeDir.split(

nimrod/setup_tools/tools.py

@@ -4,4 +4,5 @@ class Tools(Enum):
     RANDOOP='RANDOOP'
     RANDOOP_MOD='RANDOOP-MODIFIED'
     EVOSUITE='EVOSUITE'
-    DIFF_EVOSUITE='DIFFERENTIAL-EVOSUITE'
+    DIFF_EVOSUITE='DIFFERENTIAL-EVOSUITE'
+    CODELLAMA='CODELLAMA'