making SRA looper optional

Author: khoroshevskyi

MANIFEST.in

@@ -3,3 +3,4 @@ include README.md
 include docs/img/geofetch_logo.svg
 include geofetch/config_template.yaml
 include geofetch/config_processed_template.yaml
+include geofetch/looper_sra_convert.yaml

geofetch/cli.py

@@ -276,5 +276,11 @@ def _parse_cmdl(cmdl):
         help="Use just the keys defined in this module when writing out metadata.",
     )
 
+    raw_group.add_argument(
+        "--add-convert-modifier",
+        action="store_true",
+        help="Add looper SRA convert modifier to config file.",
+    )
+
     logmuse.add_logging_options(parser)
     return parser.parse_args(cmdl)

geofetch/config_template.yaml

@@ -10,51 +10,8 @@ sample_modifiers:
     # Project metadata:
     {additional_columns}
     # End of project metadata
-    SRR_files: SRA
     {pipeline_samples}
 
-  # Adding additional infromation to the project
-  derive:
-    attributes: [read1, read2, SRR_files]
-    sources:
-      SRA: "${SRABAM}/{SRR}.bam"
-      FQ: "${SRAFQ}/{SRR}.fastq.gz"
-      FQ1: "${SRAFQ}/{SRR}_1.fastq.gz"
-      FQ2: "${SRAFQ}/{SRR}_2.fastq.gz"      
-  imply:
-    - if: 
-        organism: "Mus musculus"
-      then:
-        genome: mm10
-    - if: 
-        organism: "Homo sapiens"
-      then:
-        genome: hg38          
-    - if: 
-        read_type: "PAIRED"
-      then:
-        read1: FQ1
-        read2: FQ2          
-    - if: 
-        read_type: "SINGLE"
-      then:
-        read1: FQ1
-
-project_modifiers:
-  amend:
-    sra_convert:
-      looper:
-        results_subdir: sra_convert_results
-      sample_modifiers:
-        append:
-          SRR_files: SRA
-          pipeline_interfaces: ${CODE}/geofetch/pipeline_interface_convert.yaml
-        derive:
-          attributes: [read1, read2, SRR_files]
-          sources:
-            SRA: "${SRARAW}/{SRR}.sra"
-            FQ: "${SRAFQ}/{SRR}.fastq.gz"
-            FQ1: "${SRAFQ}/{SRR}_1.fastq.gz"
-            FQ2: "${SRAFQ}/{SRR}_2.fastq.gz"
+{sra_convert}
 
 {pipeline_project}

geofetch/const.py

@@ -46,6 +46,7 @@
 
 CONFIG_PROCESSED_TEMPLATE_NAME = "config_processed_template.yaml"
 CONFIG_RAW_TEMPLATE_NAME = "config_template.yaml"
+CONFIG_SRA_TEMPLATE = "looper_sra_convert.yaml"
 
 # const for Finder:
 RETMAX = 10000000  # once it should be increased

geofetch/geofetch.py

@@ -85,6 +85,7 @@ def __init__(
         discard_soft: bool = False,
         add_dotfile: bool = False,
         disable_progressbar: bool = False,
+        add_convert_modifier: bool = False,
         opts=None,
         **kwargs,
     ):
@@ -143,6 +144,7 @@ def __init__(
         :param bam_conversion: Optional: set True to convert bam files  [Works with raw data]
         :param picard_path:  Specify a path to the picard jar, if you want to convert fastq to bam
                 [Default: $PICARD:" + safe_echo("PICARD") + "]  [Works with raw data]
+        :param add_convert_modifier: Add looper SRA convert modifier to config file.
 
         :param skip: Skip some accessions. [Default: no skip].
         :param opts: opts object [Optional]
@@ -244,7 +246,7 @@ def __init__(
         self.discard_soft = discard_soft
         self.add_dotfile = add_dotfile
         self.disable_progressbar = disable_progressbar
-
+        self.add_convert_modifier = add_convert_modifier
         self._LOGGER.info(f"Metadata folder: {self.metadata_expanded}")
 
         # Some sanity checks before proceeding
@@ -638,15 +640,15 @@ def _download_raw_data(self, run_name: str) -> NoReturn:
                     # converting sra to bam using
                     # TODO: sam-dump has a built-in prefetch. I don't have to do
                     # any of this stuff... This also solves the bad sam-dump issues.
-                    self._sra_bam_conversion1(bam_file, run_name)
+                    self._sra_to_bam_conversion_sam_dump(bam_file, run_name)
 
                     # checking if bam_file converted correctly, if not --> use fastq-dump
                     st = os.stat(bam_file)
                     if st.st_size < 100:
                         self._LOGGER.warning(
                             "Bam conversion failed with sam-dump. Trying fastq-dump..."
                         )
-                        self._sra_bam_conversion2(bam_file, run_name, self.picard_path)
+                        self._sra_to_bam_conversion_fastq_damp(bam_file, run_name, self.picard_path)
 
                 except FileNotFoundError as err:
                     self._LOGGER.info(
@@ -1145,9 +1147,15 @@ def _create_config_raw(self, proj_meta, proj_root_sample, subanot_path_yaml):
         ]
         modifiers_str = "\n    ".join(d for d in meta_list_str)
         # Write project config file
+        geofetchdir = os.path.dirname(__file__)
         if not self.config_template:
-            geofetchdir = os.path.dirname(__file__)
             self.config_template = os.path.join(geofetchdir, CONFIG_RAW_TEMPLATE_NAME)
+        if self.add_convert_modifier:
+            sra_convert_path = os.path.join(geofetchdir, CONFIG_SRA_TEMPLATE)
+            with open(sra_convert_path, "r") as template_file:
+                sra_convert_template = template_file.read()
+        else:
+            sra_convert_template = ""
         with open(self.config_template, "r") as template_file:
             template = template_file.read()
         template_values = {
@@ -1157,13 +1165,15 @@ def _create_config_raw(self, proj_meta, proj_root_sample, subanot_path_yaml):
             "pipeline_samples": self.file_pipeline_samples,
             "pipeline_project": self.file_pipeline_project,
             "additional_columns": modifiers_str,
+            "sra_convert": sra_convert_template,
         }
         for k, v in template_values.items():
             placeholder = "{" + str(k) + "}"
             template = template.replace(placeholder, str(v))
         return template
 
-    def _check_sample_name_standard(self, metadata_dict: dict) -> dict:
+    @staticmethod
+    def _check_sample_name_standard(metadata_dict: dict) -> dict:
         """
         Standardizing sample name and checking if it exists
             (This function is used for raw data)
@@ -1291,7 +1301,7 @@ def _download_SRA_file(self, run_name: str):
             )
             time.sleep(t * 2)
 
-    def _sra_bam_conversion1(self, bam_file: str, run_name: str) -> NoReturn:
+    def _sra_to_bam_conversion_sam_dump(self, bam_file: str, run_name: str) -> NoReturn:
         """
         Converting of SRA file to BAM file by using samtools function "sam-dump"
         :param str bam_file: path to BAM file that has to be created
@@ -1315,7 +1325,7 @@ def _sra_bam_conversion1(self, bam_file: str, run_name: str) -> NoReturn:
         self._LOGGER.info(f"Conversion command: {cmd}")
         run_subprocess(cmd, shell=True)
 
-    def _sra_bam_conversion2(
+    def _sra_to_bam_conversion_fastq_damp(
         self, bam_file: str, run_name: str, picard_path: str = None
     ) -> NoReturn:
         """

geofetch/looper_sra_convert.yaml

@@ -0,0 +1,45 @@
+    # Adding sra convert looper pipeline
+    SRR_files: SRA
+
+  derive:
+    attributes: [read1, read2, SRR_files]
+    sources:
+      SRA: "${SRABAM}/{SRR}.bam"
+      FQ: "${SRAFQ}/{SRR}.fastq.gz"
+      FQ1: "${SRAFQ}/{SRR}_1.fastq.gz"
+      FQ2: "${SRAFQ}/{SRR}_2.fastq.gz"
+  imply:
+    - if:
+        organism: "Mus musculus"
+      then:
+        genome: mm10
+    - if:
+        organism: "Homo sapiens"
+      then:
+        genome: hg38
+    - if:
+        read_type: "PAIRED"
+      then:
+        read1: FQ1
+        read2: FQ2
+    - if:
+        read_type: "SINGLE"
+      then:
+        read1: FQ1
+
+project_modifiers:
+  amend:
+    sra_convert:
+      looper:
+        results_subdir: sra_convert_results
+      sample_modifiers:
+        append:
+          SRR_files: SRA
+          pipeline_interfaces: ${CODE}/geofetch/pipeline_interface_convert.yaml
+        derive:
+          attributes: [read1, read2, SRR_files]
+          sources:
+            SRA: "${SRARAW}/{SRR}.sra"
+            FQ: "${SRAFQ}/{SRR}.fastq.gz"
+            FQ1: "${SRAFQ}/{SRR}_1.fastq.gz"
+            FQ2: "${SRAFQ}/{SRR}_2.fastq.gz"