Implement #334: Integrate cross-project experiment into pipeline

mubench.pipeline/tasks/configurations/configurations.py

@@ -1,12 +1,16 @@
 from typing import List
 
+from tasks.implementations.crossproject_create_index import CrossProjectCreateIndexTask
 from tasks.implementations import stats
 from tasks.implementations.checkout import CheckoutTask
 from tasks.implementations.collect_misuses import CollectMisusesTask
 from tasks.implementations.collect_projects import CollectProjectsTask
 from tasks.implementations.collect_versions import CollectVersionsTask
 from tasks.implementations.compile_misuse import CompileMisuseTask
 from tasks.implementations.compile_version import CompileVersionTask
+from tasks.implementations.crossproject_create_project_list import CrossProjectCreateProjectListTask
+from tasks.implementations.crossproject_prepare import CrossProjectPrepareTask
+from tasks.implementations.crossproject_read_index import CrossProjectReadIndexTask, CrossProjectSkipReadIndexTask
 from tasks.implementations.dataset_check_misuse import MisuseCheckTask
 from tasks.implementations.dataset_check_project import ProjectCheckTask
 from tasks.implementations.dataset_check_version import VersionCheckTask
@@ -18,6 +22,7 @@
 from tasks.implementations.load_detector import LoadDetectorTask
 from tasks.implementations.publish_findings import PublishFindingsTask
 from tasks.implementations.publish_metadata import PublishMetadataTask
+from tasks.task_runner import TaskRunner
 from utils.dataset_util import get_available_datasets
 
 
@@ -103,9 +108,10 @@ def tasks(self, config) -> List:
                                          config.java_options)
         detect = DetectProvidedCorrectUsagesTask(config.findings_path, config.force_detect, config.timeout,
                                                  config.run_timestamp)
-        return [load_detector] + CheckoutTaskConfiguration().tasks(config) + [compile_version, collect_misuses,
-                                                                              filter_misuses_without_correct_usages,
-                                                                              compile_misuse, detect]
+
+        # noinspection PyTypeChecker
+        return [load_detector] + CheckoutTaskConfiguration().tasks(config) + \
+               [compile_version, collect_misuses, filter_misuses_without_correct_usages, compile_misuse] + [detect]
 
 
 class PublishProvidedPatternsExperiment(TaskConfiguration):
@@ -132,8 +138,18 @@ def tasks(self, config) -> List:
                                              config.force_compile, config.use_tmp_wrkdir)
         load_detector = LoadDetectorTask(config.detectors_path, config.detector, config.requested_release,
                                          config.java_options)
+
+        create_index = TaskRunner([CollectMisusesTask(), CrossProjectCreateIndexTask(config.xp_index_file)])
+        read_index = CrossProjectReadIndexTask(config.xp_index_file) if config.with_xp \
+            else CrossProjectSkipReadIndexTask()
+        prepare_cross_project = [create_index, read_index,
+                                 CrossProjectPrepareTask(config.root_path, config.xp_checkouts_path,
+                                                         config.run_timestamp,
+                                                         config.max_project_sample_size, config.boa_user,
+                                                         config.boa_password)]
+
         detect = DetectAllFindingsTask(config.findings_path, config.force_detect, config.timeout, config.run_timestamp)
-        return [load_detector] + CheckoutTaskConfiguration().tasks(config) + [compile_version, detect]
+        return [load_detector] + CheckoutTaskConfiguration().tasks(config) + prepare_cross_project + [compile_version, detect]
 
 
 class PublishAllFindingsExperiment(TaskConfiguration):
@@ -161,7 +177,36 @@ def tasks(self, config) -> List:
         load_detector = LoadDetectorTask(config.detectors_path, config.detector, config.requested_release,
                                          config.java_options)
         detect = DetectAllFindingsTask(config.findings_path, config.force_detect, config.timeout, config.run_timestamp)
-        return [load_detector] + CheckoutTaskConfiguration().tasks(config) + [compile_version, detect]
+
+        create_index = TaskRunner([CollectMisusesTask(), CrossProjectCreateIndexTask(config.xp_index_file)])
+        read_index = CrossProjectReadIndexTask(config.xp_index_file) if config.with_xp \
+            else CrossProjectSkipReadIndexTask()
+        prepare_cross_project = [create_index, read_index,
+                                 CrossProjectPrepareTask(config.root_path, config.xp_checkouts_path,
+                                                         config.run_timestamp,
+                                                         config.max_project_sample_size, config.boa_user,
+                                                         config.boa_password)]
+
+        # noinspection PyTypeChecker
+        return [load_detector] + CheckoutTaskConfiguration().tasks(config) + [compile_version] + \
+            prepare_cross_project + [detect]
+
+
+class RunCrossProjectPrepare(TaskConfiguration):
+    @staticmethod
+    def mode() -> str:
+        return "checkout-xp"
+
+    def tasks(self, config) -> List:
+        create_index_tasks = [CollectProjectsTask(config.data_path), CollectVersionsTask(config.development_mode),
+                              CollectMisusesTask(), CrossProjectCreateIndexTask(config.xp_index_file)]
+        create_index = TaskRunner(create_index_tasks)
+        # noinspection PyTypeChecker
+        return [create_index,
+                CrossProjectReadIndexTask(config.xp_index_file),
+                CrossProjectPrepareTask(config.root_path, config.xp_checkouts_path, config.run_timestamp, config.max_project_sample_size, config.boa_user,
+                                        config.boa_password),
+                CrossProjectCreateProjectListTask(config.root_path, config.xp_index_file, config.xp_checkouts_path)]
 
 
 class PublishBenchmarkExperiment(TaskConfiguration):

mubench.pipeline/tasks/implementations/crossproject_create_index.py

@@ -0,0 +1,20 @@
+import os
+
+from data.misuse import Misuse
+from data.project import Project
+from data.project_version import ProjectVersion
+from utils.io import safe_open
+
+
+class CrossProjectCreateIndexTask:
+    def __init__(self, index_file: str):
+        self.index_file = index_file
+
+        if os.path.exists(index_file):
+            os.remove(index_file)
+
+    def run(self, project: Project, version: ProjectVersion, misuse: Misuse):
+        print("{}\t{}\t{}\t{}\t{}\t{}\t{}".format(project.id, version.version_id, misuse.misuse_id,
+                                                  ':'.join(version.source_dirs),
+                                                  misuse.location.file, misuse.location.method,
+                                                  "\t".join(misuse.apis)), file=safe_open(self.index_file + '-' + version.id, "a"))

mubench.pipeline/tasks/implementations/crossproject_create_project_list.py

@@ -0,0 +1,39 @@
+import csv
+import logging
+import os
+from os.path import join
+
+from utils.io import open_yamls_if_exists, write_yaml
+from utils.shell import Shell
+
+
+class CrossProjectCreateProjectListTask:
+    def __init__(self, root_path: str, index_file: str, base_checkout_path: str):
+        self.root_path = root_path
+        self.index_file = index_file
+        self.base_checkout_path = base_checkout_path
+
+    def run(self):
+        logger = logging.getLogger("tasks.cross_project_create_project_list")
+        example_projects_by_API = {}
+        with open(self.index_file) as index:
+            for row in csv.reader(index, delimiter="\t"):
+                # skip blank lines, e.g., on trailing newline
+                if not row:
+                    continue
+
+                target_type = row[6]
+                try:
+                    if target_type not in example_projects_by_API:
+                        logger.info("Preparing examples for type: %s...", target_type)
+                        target_example_file = os.path.join(self.base_checkout_path, target_type + ".yml")
+                        example_projects = {}
+                        with open_yamls_if_exists(target_example_file) as projects:
+                            for project in projects:
+                                hash = Shell.exec("cd \"{}\"; git rev-parse HEAD".format(join(self.root_path, project["path"])))
+                                example_projects[project["url"]] = hash.strip()
+                        example_projects_by_API[target_type] = example_projects
+                except Exception as error:
+                    logger.exception("failed", exc_info=error)
+
+        write_yaml(example_projects_by_API, join(self.base_checkout_path, "example_projects_by_API.yml"))