Merge pull request #113 from opain/dev

Dev

.github/workflows/run_quick_tests.yaml

@@ -2,25 +2,25 @@ name: Run quick tests of pipeline
 
 on:
   pull_request:
-    types: [closed]
     branches:
       - master
   workflow_dispatch:
 
 jobs:
   run-tests:
-    if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
+    container:
+      image: rocker/r-ver:4.1.0
+      options: --privileged
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
 
-      - name: Set up R
-        uses: r-lib/actions/setup-r@v2
-
-      - name: Setup singularity
+      - name: Setup Singularity
         uses: eWaterCycle/setup-singularity@v7
+        with:
+          singularity-version: 3.8.3
 
       - name: Download Singularity container
         run: |
@@ -29,28 +29,25 @@ jobs:
       - name: Set environment variables
         run: |
           echo "SIF_FILE=$GITHUB_WORKSPACE/genopred_pipeline_latest.sif" >> $GITHUB_ENV
-          echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
-
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
+          else
+            echo "BRANCH_NAME=${{ github.ref_name }}" >> $GITHUB_ENV
+          fi
+          
       - name: Install R dependencies
         run: |
           Rscript -e 'install.packages("testthat", repos="https://cran.rstudio.com/")'
           Rscript -e 'install.packages("data.table", repos="https://cran.rstudio.com/")'
+          Rscript -e 'install.packages("R.utils", repos="https://cran.rstudio.com/")'
 
       - name: Run tests
         run: |
-          mkdir -p test_results
-          Rscript -e "testthat::test_dir('pipeline/tests/testthat', reporter = 'junit')" > test_results/results.xml
+          Rscript -e "testthat::test_dir('pipeline/tests/testthat')"
         env:
           SIF_FILE: ${{ env.SIF_FILE }}
           BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
-      - name: Upload test results
-        if: always()
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-results
-          path: test_results/results.xml
-
       - name: Setup tmate session
         if: ${{ failure() }}
         uses: mxschmitt/action-tmate@v3

pipeline/misc/dev/test_data/output/example_plink2/ancestry/example_plink2.Ancestry.log

@@ -3,7 +3,7 @@
 # For questions contact Oliver Pain ([email protected])
 #################################################################
 # Repository: GenoPred
-# Version (tag): v2.2.2-95-g19dbe57
+# Version (tag): v2.2.2-102-g0438efa
 ---------------
  Parameter        Value                                                                    
  target_plink_chr misc/dev/test_data/output/example_plink2/geno/example_plink2.ref.chr     
@@ -23,7 +23,7 @@
  help             FALSE                                                                    
  out_dir          misc/dev/test_data/output/example_plink2/ancestry/                       
 ---------------
-Analysis started at 2024-07-25 09:40:29
+Analysis started at 2024-07-25 15:07:05
 Lowering prob_thresh parameter to 0.5 for testing.
 Target sample size is <100 so only checking genotype missingness.
 587 variants match between target and reference after QC.
@@ -47,5 +47,5 @@ N per group based on model:
  MID        0
  Unassigned 2
 ----------
-Analysis finished at 2024-07-25 09:40:44 
-Analysis duration was 15.33 secs 
+Analysis finished at 2024-07-25 15:07:19 
+Analysis duration was 14.27 secs 

pipeline/misc/dev/test_data/output/example_plink2/geno/example_plink2.ref.chr22.format_target.log

@@ -3,7 +3,7 @@
 # For questions contact Oliver Pain ([email protected])
 #################################################################
 # Repository: GenoPred
-# Version (tag): v2.2.2-95-g19dbe57
+# Version (tag): v2.2.2-102-g0438efa
 ---------------
  Parameter Value                                                                 
  target    misc/dev/test_data/target/example.chr22                               
@@ -15,7 +15,7 @@
  help      FALSE                                                                 
  out_dir   misc/dev/test_data/output/example_plink2/geno/                        
 ---------------
-Analysis started at 2024-07-25 09:40:19
+Analysis started at 2024-07-25 15:07:00
 Reading in reference SNP data.
 Reference data contains 1000 variants.
 Reading in target SNP data.
@@ -26,5 +26,5 @@ GRCh38 match: 0%
 Target contains 1000 reference variants.
 Removing 0 duplicate variants - May have IUPAC NA.
 Inserting missing reference variants.
-Analysis finished at 2024-07-25 09:40:23 
-Analysis duration was 4.25 secs 
+Analysis finished at 2024-07-25 15:07:01 
+Analysis duration was 0.7 secs 

pipeline/misc/dev/test_data/output/example_plink2/geno/example_plink2.ref.chr22.log

@@ -1,24 +1,24 @@
-PLINK v2.00a5LM 64-bit Intel (23 Sep 2023)
+PLINK v2.00a5.12LM 64-bit Intel (25 Jun 2024)
 Options in effect:
-  --bfile /scratch/prj/oliverpainfel/tmp/RtmpClbFKL/ref_targ
+  --bfile /scratch/prj/oliverpainfel/tmp/RtmpqFvaS6/ref_targ
   --make-pgen
   --memory 5000
   --out misc/dev/test_data/output/example_plink2/geno/example_plink2.ref.chr22
-  --remove /scratch/prj/oliverpainfel/tmp/RtmpClbFKL/REF.psam
+  --remove /scratch/prj/oliverpainfel/tmp/RtmpqFvaS6/REF.psam
   --threads 1
 
 Hostname: erc-hpc-comp179
-Working directory: /scratch/prj/oliverpainfel/Software/MyGit/GenoPred/pipeline
-Start time: Thu Jul 25 09:40:23 2024
+Working directory: /tools/GenoPred/pipeline
+Start time: Thu Jul 25 15:07:01 2024
 
-Random number seed: 1721896823
-1031702 MiB RAM detected, ~1018874 available; reserving 5000 MiB for main
+Random number seed: 1721916421
+1031702 MiB RAM detected, ~1018552 available; reserving 5000 MiB for main
 workspace.
 Using 1 compute thread.
 3325 samples (1573 females, 1752 males; 3325 founders) loaded from
-/scratch/prj/oliverpainfel/tmp/RtmpClbFKL/ref_targ.fam.
+/scratch/prj/oliverpainfel/tmp/RtmpqFvaS6/ref_targ.fam.
 1000 variants loaded from
-/scratch/prj/oliverpainfel/tmp/RtmpClbFKL/ref_targ.bim.
+/scratch/prj/oliverpainfel/tmp/RtmpqFvaS6/ref_targ.bim.
 Note: No phenotype data present.
 --remove: 12 samples remaining.
 12 samples (5 females, 7 males; 12 founders) remaining after main filters.
@@ -32,4 +32,4 @@ Writing
 misc/dev/test_data/output/example_plink2/geno/example_plink2.ref.chr22.pgen ...
 done.
 
-End time: Thu Jul 25 09:40:23 2024
+End time: Thu Jul 25 15:07:01 2024

pipeline/misc/dev/test_data/output/example_plink2/pcs/projected/AFR/example_plink2-AFR.log

@@ -3,7 +3,7 @@
 # For questions contact Oliver Pain ([email protected])
 #################################################################
 # Repository: GenoPred
-# Version (tag): v2.2.2-95-g19dbe57
+# Version (tag): v2.2.2-102-g0438efa
 ---------------
  Parameter        Value                                                                                      
  target_plink_chr misc/dev/test_data/output/example_plink2/geno/example_plink2.ref.chr                       
@@ -19,9 +19,9 @@
  help             FALSE                                                                                      
  output_dir       misc/dev/test_data/output/example_plink2/pcs/projected/AFR/                                
 ---------------
-Analysis started at 2024-07-25 09:41:24
+Analysis started at 2024-07-25 15:07:20
 Calculating polygenic scores in the target sample.
 Scaling target polygenic scores to the reference.
 Saved polygenic scores to: misc/dev/test_data/output/example_plink2/pcs/projected/AFR/example_plink2-AFR.profiles.
-Analysis finished at 2024-07-25 09:41:25 
-Analysis duration was 0.92 secs 
+Analysis finished at 2024-07-25 15:07:20 
+Analysis duration was 0.22 secs 

pipeline/misc/dev/test_data/output/example_plink2/pcs/projected/CSA/example_plink2-CSA.log

@@ -3,7 +3,7 @@
 # For questions contact Oliver Pain ([email protected])
 #################################################################
 # Repository: GenoPred
-# Version (tag): v2.2.2-95-g19dbe57
+# Version (tag): v2.2.2-102-g0438efa
 ---------------
  Parameter        Value                                                                                      
  target_plink_chr misc/dev/test_data/output/example_plink2/geno/example_plink2.ref.chr                       
@@ -19,9 +19,9 @@
  help             FALSE                                                                                      
  output_dir       misc/dev/test_data/output/example_plink2/pcs/projected/CSA/                                
 ---------------
-Analysis started at 2024-07-25 09:41:14
+Analysis started at 2024-07-25 15:07:22
 Calculating polygenic scores in the target sample.
 Scaling target polygenic scores to the reference.
 Saved polygenic scores to: misc/dev/test_data/output/example_plink2/pcs/projected/CSA/example_plink2-CSA.profiles.
-Analysis finished at 2024-07-25 09:41:16 
-Analysis duration was 1.51 secs 
+Analysis finished at 2024-07-25 15:07:22 
+Analysis duration was 0.22 secs 

pipeline/misc/dev/test_data/output/example_plink2/pcs/projected/EAS/example_plink2-EAS.log

@@ -3,7 +3,7 @@
 # For questions contact Oliver Pain ([email protected])
 #################################################################
 # Repository: GenoPred
-# Version (tag): v2.2.2-95-g19dbe57
+# Version (tag): v2.2.2-102-g0438efa
 ---------------
  Parameter        Value                                                                                      
  target_plink_chr misc/dev/test_data/output/example_plink2/geno/example_plink2.ref.chr                       
@@ -19,9 +19,9 @@
  help             FALSE                                                                                      
  output_dir       misc/dev/test_data/output/example_plink2/pcs/projected/EAS/                                
 ---------------
-Analysis started at 2024-07-25 09:41:07
+Analysis started at 2024-07-25 15:07:21
 Calculating polygenic scores in the target sample.
 Scaling target polygenic scores to the reference.
 Saved polygenic scores to: misc/dev/test_data/output/example_plink2/pcs/projected/EAS/example_plink2-EAS.profiles.
-Analysis finished at 2024-07-25 09:41:09 
-Analysis duration was 2.18 secs 
+Analysis finished at 2024-07-25 15:07:22 
+Analysis duration was 0.24 secs 

pipeline/misc/dev/test_data/output/example_plink2/pcs/projected/EUR/example_plink2-EUR.log

@@ -3,7 +3,7 @@
 # For questions contact Oliver Pain ([email protected])
 #################################################################
 # Repository: GenoPred
-# Version (tag): v2.2.2-95-g19dbe57
+# Version (tag): v2.2.2-102-g0438efa
 ---------------
  Parameter        Value                                                                                      
  target_plink_chr misc/dev/test_data/output/example_plink2/geno/example_plink2.ref.chr                       
@@ -19,9 +19,9 @@
  help             FALSE                                                                                      
  output_dir       misc/dev/test_data/output/example_plink2/pcs/projected/EUR/                                
 ---------------
-Analysis started at 2024-07-25 09:40:59
+Analysis started at 2024-07-25 15:07:21
 Calculating polygenic scores in the target sample.
 Scaling target polygenic scores to the reference.
 Saved polygenic scores to: misc/dev/test_data/output/example_plink2/pcs/projected/EUR/example_plink2-EUR.profiles.
-Analysis finished at 2024-07-25 09:41:01 
-Analysis duration was 1.79 secs 
+Analysis finished at 2024-07-25 15:07:21 
+Analysis duration was 0.24 secs 

pipeline/misc/dev/test_data/output/example_plink2/pgs/AFR.log

@@ -3,7 +3,7 @@
 # For questions contact Oliver Pain ([email protected])
 #################################################################
 # Repository: GenoPred
-# Version (tag): v2.2.2-95-g19dbe57
+# Version (tag): v2.2.2-102-g0438efa
 ---------------
  Parameter  Value                                           
  config     misc/dev/test_data/config/config.yaml           
@@ -16,13 +16,13 @@
  help       FALSE                                           
  output_dir misc/dev/test_data/output/example_plink2/pgs/AFR
 ---------------
-Analysis started at 2024-07-25 11:37:16
+Analysis started at 2024-07-25 15:07:25
 Processing 3 score files.
 Aggregating score files in 1 batches.
 Aggregating batched score files.
 Reading in scale files.
 Calculating polygenic scores in the target sample.
 Scaling target polygenic scores to the reference.
 Saved polygenic scores.
-Analysis finished at 2024-07-25 11:37:16 
-Analysis duration was 0.34 secs 
+Analysis finished at 2024-07-25 15:07:26 
+Analysis duration was 0.35 secs 

pipeline/misc/dev/test_data/output/example_plink2/pgs/CSA.log

@@ -3,7 +3,7 @@
 # For questions contact Oliver Pain ([email protected])
 #################################################################
 # Repository: GenoPred
-# Version (tag): v2.2.2-95-g19dbe57
+# Version (tag): v2.2.2-102-g0438efa
 ---------------
  Parameter  Value                                           
  config     misc/dev/test_data/config/config.yaml           
@@ -16,13 +16,13 @@
  help       FALSE                                           
  output_dir misc/dev/test_data/output/example_plink2/pgs/CSA
 ---------------
-Analysis started at 2024-07-25 11:37:17
+Analysis started at 2024-07-25 15:07:26
 Processing 3 score files.
 Aggregating score files in 1 batches.
 Aggregating batched score files.
 Reading in scale files.
 Calculating polygenic scores in the target sample.
 Scaling target polygenic scores to the reference.
 Saved polygenic scores.
-Analysis finished at 2024-07-25 11:37:19 
-Analysis duration was 1.99 secs 
+Analysis finished at 2024-07-25 15:07:26 
+Analysis duration was 0.34 secs 

pipeline/misc/dev/test_data/output/example_plink2/pgs/EAS.log

@@ -3,7 +3,7 @@
 # For questions contact Oliver Pain ([email protected])
 #################################################################
 # Repository: GenoPred
-# Version (tag): v2.2.2-95-g19dbe57
+# Version (tag): v2.2.2-102-g0438efa
 ---------------
  Parameter  Value                                           
  config     misc/dev/test_data/config/config.yaml           
@@ -16,13 +16,13 @@
  help       FALSE                                           
  output_dir misc/dev/test_data/output/example_plink2/pgs/EAS
 ---------------
-Analysis started at 2024-07-25 11:37:15
+Analysis started at 2024-07-25 15:07:24
 Processing 3 score files.
 Aggregating score files in 1 batches.
 Aggregating batched score files.
 Reading in scale files.
 Calculating polygenic scores in the target sample.
 Scaling target polygenic scores to the reference.
 Saved polygenic scores.
-Analysis finished at 2024-07-25 11:37:15 
-Analysis duration was 0.39 secs 
+Analysis finished at 2024-07-25 15:07:25 
+Analysis duration was 0.32 secs 

pipeline/misc/dev/test_data/output/example_plink2/pgs/EUR.log

@@ -3,7 +3,7 @@
 # For questions contact Oliver Pain ([email protected])
 #################################################################
 # Repository: GenoPred
-# Version (tag): v2.2.2-95-g19dbe57
+# Version (tag): v2.2.2-102-g0438efa
 ---------------
  Parameter  Value                                           
  config     misc/dev/test_data/config/config.yaml           
@@ -16,13 +16,13 @@
  help       FALSE                                           
  output_dir misc/dev/test_data/output/example_plink2/pgs/EUR
 ---------------
-Analysis started at 2024-07-25 11:37:14
+Analysis started at 2024-07-25 15:07:24
 Processing 3 score files.
 Aggregating score files in 1 batches.
 Aggregating batched score files.
 Reading in scale files.
 Calculating polygenic scores in the target sample.
 Scaling target polygenic scores to the reference.
 Saved polygenic scores.
-Analysis finished at 2024-07-25 11:37:15 
-Analysis duration was 0.48 secs 
+Analysis finished at 2024-07-25 15:07:24 
+Analysis duration was 0.35 secs 

pipeline/misc/dev/test_data/output/reference/benchmarks/ancestry_inference_i-example_plink2.txt

@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-17.9874	0:00:17	441.39	4642.98	437.00	437.74	0.00	1.20	64.70	11.98
+14.6503	0:00:14	451.30	516539.48	447.39	449.39	0.00	1.07	80.62	12.61

pipeline/misc/dev/test_data/output/reference/benchmarks/ancestry_reporter-example_plink2.txt

@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-0.3615	0:00:00	3.00	9.88	0.04	0.58	0.00	0.00	0.00	0.00
+0.3222	0:00:00	0	2.03	0.12	0.27	0.00	0.00	0.00	0.00

pipeline/misc/dev/test_data/output/reference/benchmarks/format_target_i-example_plink2-22.txt

@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-4.6663	0:00:04	110.76	5348.30	103.86	104.58	0.00	3.69	4.52	0.53
+1.0635	0:00:01	70.83	158.56	67.31	69.17	0.00	0.00	0.00	0.31

pipeline/misc/dev/test_data/output/reference/benchmarks/indiv_report_i-example_plink2-4_EAS.4_EAS.txt

@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-5.0812	0:00:05	367.09	858.32	360.86	361.77	0.00	1.75	78.74	4.30
+5.4381	0:00:05	375.84	769.37	372.47	374.81	0.00	0.19	72.60	4.23

pipeline/misc/dev/test_data/output/reference/benchmarks/pc_projection_i-example_plink2-AFR.txt

@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1.3357	0:00:01	70.16	241.25	64.66	65.35	0.00	0.05	8.31	0.31
+0.5640	0:00:00	72.84	227.62	67.83	70.07	0.00	0.04	0.00	0.28

pipeline/misc/dev/test_data/output/reference/benchmarks/pc_projection_i-example_plink2-CSA.txt

@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-2.0147	0:00:02	70.16	241.26	64.70	65.41	0.00	0.04	5.55	0.31
+0.5473	0:00:00	2.00	4.36	0.08	1.40	0.00	0.00	0.00	0.00

pipeline/misc/dev/test_data/output/reference/benchmarks/pc_projection_i-example_plink2-EAS.txt

@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-2.7472	0:00:02	69.16	259.61	63.95	64.67	0.00	0.03	3.27	0.29
+0.5695	0:00:00	70.84	222.18	67.38	69.07	0.00	0.03	0.00	0.28

pipeline/misc/dev/test_data/output/reference/benchmarks/pc_projection_i-example_plink2-EUR.txt

@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-2.6030	0:00:02	72.16	260.31	65.09	65.89	0.00	0.04	4.14	0.31
+0.5673	0:00:00	70.52	222.23	67.42	69.29	0.00	0.03	0.00	0.28

pipeline/misc/dev/test_data/output/reference/benchmarks/prep_pgs_lassosum_i-BODY04.txt

@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-27.3731	0:00:27	253.16	568.40	240.51	244.92	0.00	0.57	4.14	1.34
+3.1643	0:00:03	264.84	550.32	250.21	257.22	0.00	0.26	28.43	1.22

pipeline/misc/dev/test_data/output/reference/benchmarks/prep_pgs_ptclump_i-BODY04.txt

@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-0.8348	0:00:00	68.16	259.14	63.45	63.98	0.00	0.07	0.00	0.28
+1.0572	0:00:01	69.93	213.49	66.80	68.63	0.00	0.06	0.00	0.28

pipeline/misc/dev/test_data/output/reference/benchmarks/sample_report_i-example_plink2.txt

@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-2.5578	0:00:02	189.13	480.65	184.38	185.05	0.00	0.21	60.74	1.85
+3.3662	0:00:03	283.84	1049383.44	278.54	281.12	0.00	0.18	47.47	1.86