Containerize all nextflow processes

.github/workflows/build.yml

@@ -86,7 +86,7 @@ jobs:
         uses: adamrtalbot/detect-nf-test-changes@v0.0.6
         with:
           head: ${{ github.sha }}
-          base: ${{ github.base_ref }}
+          base: origin/${{ github.base_ref }}
           include: .github/include.yml
       - name: print list of nf-test files
         run: |

.gitignore

@@ -39,6 +39,8 @@ log/
 .nf-test/
 results/
 output
+.singularity/
+submit_slurm.sh
 
 # tmp ignore testing fastqs
 test_dir/*/*/*.gz

CHANGELOG.md

@@ -1,5 +1,10 @@
 ## SINCLAIR development version
 
+- All nextflow processes now use containers rather than environment modules for dependencies. (#95, @kelly-sovacool)
+
+### Bug fixes
+
+- Creates cache for celldex reference downloads in working directory (#203, @wong-nw)
 - Set main job memory to 2 GB for slurm mode. (#205, @kelly-sovacool)
 
 ## SINCLAIR 0.3.5

assets/slurm_header_biowulf.sh

@@ -6,7 +6,7 @@
 #SBATCH -J "sinclair"
 #SBATCH --mail-type=BEGIN,END,FAIL
 #SBATCH --output "log/slurm_%j.log"
-#SBATCH --output "log/slurm_%j.log"
+#SBATCH --error "log/slurm_%j.log"
 
 module load ccbrpipeliner
 module load nextflow

assets/slurm_header_frce.sh

@@ -6,7 +6,7 @@
 #SBATCH -J "sinclair"
 #SBATCH --mail-type=BEGIN,END,FAIL
 #SBATCH --output "log/slurm_%j.log"
-#SBATCH --output "log/slurm_%j.log"
+#SBATCH --error "log/slurm_%j.log"
 
 module load nextflow
 NXF_SINGULARITY_CACHEDIR=/mnt/projects/CCBR-Pipelines/SIFs

bin/batch_correction_cca.Rmd

@@ -8,58 +8,44 @@ editor_options:
 params:
   species: "hg38"
   gid: "group1-group2"
-  mergedObj: "/data/sevillas2/sinclair/dev/results/seurat/merge/group1-group2_seurat_merged.rds"
+  mergedObj: "results/seurat/merge/group1-group2_seurat_merged.rds"
   resolution_list: "0.1,0.2,0.3,0.5,0.6,0.8,1"
   npcs: 50
   vars_to_regress: NULL
-  Rlib_dir: "/data/CCBR_Pipeliner/db/PipeDB/Rlibrary_4.3_scRNA_RHEL8/"
-  Rpkg_config: "/data/CCBR_Pipeliner/Pipelines/SINCLAIR/dev/conf/Rpack.config"
-  scRNA_functions: "/data/CCBR_Pipeliner/Pipelines/SINCLAIR/dev/bin/scRNA_functions.R"
+  scRNA_functions: "bin/scRNA_functions.R"
+  celldex_cache: NULL
   testing: "Y"
 ---
 
 
-```{r, prep_args,  message=FALSE}
+```{r setup,  message=FALSE}
+options(future.globals.maxSize = 96000 * 1024^2)
 # set up params
 species <- params$species
 gid <- params$gid
 mergedObj <- params$mergedObj
 resolution <- as.numeric(strsplit(params$resolution_list, ",")[[1]])
 npcs <- as.numeric(params$npcs)
+
 vars_to_regress <- params$vars_to_regress
 vars_to_regress_list <- if (is.null(vars_to_regress) || toupper(vars_to_regress) == "NULL") {
   NULL
 } else {
   unlist(strsplit(gsub(" ", "", vars_to_regress), ","))
 }
-Rlib_dir <- params$Rlib_dir
-Rpkg_config <- params$Rpkg_config
-testing <- params$testing
-scRNA_functions <- params$scRNA_functions
-```
-
-```{r, handle_pkg, message=FALSE}
-# source functions
-source(scRNA_functions)
 
-# set library dir, load this and remove any other dirs to avoid confusion
-# between personally created pkgs and the pipeline package
-## saving old path "/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library"
-print(paste0("Using the lib.loc location: ", Rlib_dir))
-assign(".lib.loc", Rlib_dir, envir = environment(.libPaths))
-.libPaths()
-
-# read in package info
-pkg_df <- read.csv(Rpkg_config)
-pkg_df <- subset(pkg_df, cca == "Y")
-pkg_df
-
-# for each package check installation, if present then load library
-scRNA_handle_packages(pkg_df)
+celldex_cache <- params$celldex_cache
+testing <- params$testing
 
-# additional options
-# tinytex::install_tinytex(force = TRUE)
-options(future.globals.maxSize = 96000 * 1024^2)
+source(params$scRNA_functions)
+library(Seurat)
+library(SingleR)
+library(tinytex)
+library(AnnotationDbi)
+library(org.Hs.eg.db)
+library(org.Mm.eg.db)
+library(Orthology.eg.db)
+library(harmony)
 ```
 
 ```{r, processing, message=FALSE}
@@ -74,7 +60,7 @@ so[["RNA"]] <- split(so[["RNA"]], f = so$Sample)
 # integrate
 so_corrected <- MAIN_BATCH_CORRECTION(so, npcs, species, resolution,
   method_in = "CCAIntegration", reduction_in = "CCA",
-  v_list = vars_to_regress_list
+  v_list = vars_to_regress_list, cache_path = celldex_cache
 )
 ```
 

bin/batch_correction_harmony.Rmd

@@ -7,56 +7,43 @@ editor_options:
   chunk_output_type: console
 params:
   gid: "group1_group2"
-  mergedObj: "/data/sevillas2/sinclair/dev/results/seurat/merge/group1-group2_seurat_merged.rds"
+  mergedObj: "results/seurat/merge/group1-group2_seurat_merged.rds"
   species: "hg38"
   resolution_list: "0.1,0.2,0.3,0.5,0.6,0.8,1"
   npcs: "50"
   vars_to_regress: NULL
-  Rlib_dir: "/data/CCBR_Pipeliner/db/PipeDB/Rlibrary_4.3_scRNA_RHEL8/"
-  Rpkg_config: "/data/CCBR_Pipeliner/Pipelines/SINCLAIR/dev/conf/Rpack.config"
-  scRNA_functions: "/data/CCBR_Pipeliner/Pipelines/SINCLAIR/dev/bin/scRNA_functions.R"
+  scRNA_functions: "bin/scRNA_functions.R"
+  celldex_cache: NULL
   testing: "Y"
 ---
 
-```{r, prep_args,  message=FALSE, echo=FALSE, include=FALSE}
+```{r setup,  message=FALSE, echo=FALSE, include=FALSE}
+options(future.globals.maxSize = 96000 * 1024^2)
 # set up params
 species <- params$species
 gid <- params$gid
 mergedObj <- params$mergedObj
 resolution <- as.numeric(strsplit(params$resolution_list, ",")[[1]])
 npcs <- as.numeric(params$npcs)
+
+source(params$scRNA_functions)
+library(Seurat)
+library(SeuratWrappers)
+library(SingleR)
+library(tinytex)
+library(AnnotationDbi)
+library(org.Hs.eg.db)
+library(org.Mm.eg.db)
+library(Orthology.eg.db)
+library(harmony)
 vars_to_regress <- params$vars_to_regress
 vars_to_regress_list <- if (is.null(vars_to_regress) || toupper(vars_to_regress) == "NULL") {
   NULL
 } else {
   unlist(strsplit(gsub(" ", "", vars_to_regress), ","))
 }
-Rlib_dir <- params$Rlib_dir
-Rpkg_config <- params$Rpkg_config
-scRNA_functions <- params$scRNA_functions
-```
-
-```{r, handle_pkg, message=FALSE}
-# source functions
-source(scRNA_functions)
-
-# set library dir, load this and remove any other dirs to avoid confusion
-# between personally created pkgs and the pipeline package
-## saving old path "/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library"
-print(paste0("Using the lib.loc location: ", Rlib_dir))
-assign(".lib.loc", Rlib_dir, envir = environment(.libPaths))
-.libPaths()
-
-# read in package info
-pkg_df <- read.csv(Rpkg_config)
-pkg_df <- subset(pkg_df, harmony == "Y")
-pkg_df
-
-# for each package check installation, if present then load library
-scRNA_handle_packages(pkg_df)
+celldex_cache <- params$celldex_cache
 
-# additional options
-# tinytex::install_tinytex(force = TRUE)
 options(future.globals.maxSize = 96000 * 1024^2)
 ```
 
@@ -72,7 +59,7 @@ so[["RNA"]] <- split(so[["RNA"]], f = so$Sample)
 # integrate
 so_corrected <- MAIN_BATCH_CORRECTION(so, npcs, species, resolution,
   method_in = "HarmonyIntegration", reduction_in = "harmony",
-  v_list = vars_to_regress_list
+  v_list = vars_to_regress_list, cache_path = celldex_cache
 )
 ```