Support CRAM archival mode

Author: bbimber

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AbstractAlignmentStepProvider.java

@@ -34,6 +34,7 @@ abstract public class AbstractAlignmentStepProvider<StepType extends AlignmentSt
     public static String SUPPORT_MERGED_UNALIGNED = "supportsMergeUnaligned";
     public static String COLLECT_WGS_METRICS = "collectWgsMetrics";
     public static String CONVERT_TO_CRAM = "convertToCram";
+    public static String CRAM_ARCHIVAL_MODE = "doCramArchivalMode";
     public static String COLLECT_WGS_METRICS_NON_ZERO = "collectWgsMetricsNonZero";
     public static String DISCARD_BAM = "discardBam";
 
@@ -116,6 +117,10 @@ private static List<ToolParameterDescriptor> getParamList(List<ToolParameterDesc
             put("checked", false);
         }}, false));
 
+        parameters.add(ToolParameterDescriptor.create("doCramArchivalMode", "CRAM Archival Mode", "If selected, the CRAM will undergo additional compression to save space. This is lossy and may not be compatible with all downstream tools. See samtools view --output-fmt-option archive", "checkbox", new JSONObject(){{
+            put("checked", false);
+        }}, false));
+
         parameters.add(ToolParameterDescriptor.create(ALIGNMENT_MODE_PARAM, "Alignment Mode", "If your readset has more than one pair of FASTQs, there pipeline can either align each pair sequentially (and then merge these BAMs), or merge the pairs of FASTQs first and then perform alignment once.  The default is to align each pair of FASTQs separately; however, some pipelines like STAR require the latter.", "ldk-simplecombo", new JSONObject(){{
             put("storeValues", ALIGNMENT_MODE.ALIGN_THEN_MERGE.name() + ";" + ALIGNMENT_MODE.MERGE_THEN_ALIGN.name());
             put("value", alignmentMode.name());

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SamtoolsCramConverter.java

@@ -18,25 +18,40 @@ public SamtoolsCramConverter(Logger log)
         super(log);
     }
 
-    public File convert(File inputBam, File outputCram, File gzippedFasta, boolean doIndex, @Nullable Integer threads) throws PipelineJobException
+    public File convert(File inputBam, File outputCram, File gzippedFasta, boolean doIndex, @Nullable Integer threads, boolean archivalMode) throws PipelineJobException
     {
         getLogger().info("Converting SAM/BAM to CRAM: " + inputBam.getPath());
 
         List<String> params = new ArrayList<>();
         params.add(getSamtoolsPath().getPath());
         params.add("view");
 
-        params.add("-C");
+        params.add("--output-fmt");
+        params.add("cram,version=3.0");
 
         params.add("-o");
         params.add(outputCram.getPath());
 
+        // CRAM does, however, have an optional archive settings mode (samtools view  ...)
+        // which is a lossy compression, doing things like removing read names, removing additional accessory fields, and additional compression of quality scores.
+        // In all cases, the base sequence of the reads is preserved: https://www.htslib.org/doc/samtools.html
+        if (archivalMode)
+        {
+            params.add("--output-fmt-option");
+            params.add("archive");
+        }
+
         params.add("-T");
         params.add(gzippedFasta.getPath());
 
+        if (doIndex)
+        {
+            params.add("--write-index");
+        }
+
         if (threads != null)
         {
-            params.add("--threads");
+            params.add("-@");
             params.add(String.valueOf(threads));
         }
 
@@ -49,11 +64,6 @@ public File convert(File inputBam, File outputCram, File gzippedFasta, boolean d
             throw new PipelineJobException("Missing output: " + outputCram.getPath());
         }
 
-        if (doIndex)
-        {
-            doIndex(outputCram, threads);
-        }
-
         return outputCram;
     }
 

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ConvertToCramHandler.java

@@ -42,6 +42,9 @@ public ConvertToCramHandler()
                 ToolParameterDescriptor.create("replaceOriginal", "Replace Original File", "If selected, the input BAM will be deleted and the database record will be switched to use this filepath.", "checkbox", new JSONObject(){{
                     put("checked", true);
                 }}, true),
+                ToolParameterDescriptor.create("doCramArchivalMode", "CRAM Archival Mode", "If selected, the CRAM will undergo additional compression to save space. This is lossy and may not be compatible with all downstream tools. See samtools view --output-fmt-option archive", "checkbox", new JSONObject(){{
+                    put("checked", false);
+                }}, false),
                 ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook.  This is primarily useful if submitting a large batch of files to process separately. This only applies if 'Run Separately' is selected.", "checkbox", new JSONObject(){{
                     put("checked", true);
                 }}, true)
@@ -103,6 +106,7 @@ public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport
         public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext ctx) throws UnsupportedOperationException, PipelineJobException
         {
             boolean replaceOriginal = ctx.getParams().optBoolean("replaceOriginal", false);
+            boolean doCramArchivalMode = ctx.getParams().optBoolean("doCramArchivalMode", false);
             ctx.getLogger().info("Replace input BAM: " + replaceOriginal);
 
             Integer threads = SequencePipelineService.get().getMaxThreads(ctx.getLogger());
@@ -124,7 +128,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
                 }
                 else
                 {
-                    new SamtoolsCramConverter(ctx.getLogger()).convert(so.getFile(), cram, genome.getWorkingFastaFileGzipped(), true, threads);
+                    new SamtoolsCramConverter(ctx.getLogger()).convert(so.getFile(), cram, genome.getWorkingFastaFileGzipped(), true, threads, doCramArchivalMode);
                 }
 
                 checkCramAndIndex(so);

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentTask.java

@@ -1145,7 +1145,9 @@ else if (step.expectToCreateNewBam())
 
         // optional convert to CRAM:
         ToolParameterDescriptor cramParam = alignmentStep.getProvider().getParameterByName(AbstractAlignmentStepProvider.CONVERT_TO_CRAM);
+        ToolParameterDescriptor cramArchivalParam = alignmentStep.getProvider().getParameterByName(AbstractAlignmentStepProvider.CRAM_ARCHIVAL_MODE);
         boolean doCramConvert = cramParam != null && cramParam.extractValue(getJob(), alignmentStep.getProvider(), alignmentStep.getStepIdx(), Boolean.class, false);
+        boolean doArchival = cramArchivalParam != null && cramArchivalParam.extractValue(getJob(), alignmentStep.getProvider(), alignmentStep.getStepIdx(), Boolean.class, false);
         if (doCramConvert)
         {
             getJob().getLogger().info("BAM will be converted to CRAM");
@@ -1154,7 +1156,7 @@ else if (step.expectToCreateNewBam())
             Integer threads = SequenceTaskHelper.getMaxThreads(getJob());
             if (!cramFileIdx.exists())
             {
-                new SamtoolsCramConverter(getJob().getLogger()).convert(renamedBam, cramFile, referenceGenome.getWorkingFastaFileGzipped(), true, threads);
+                new SamtoolsCramConverter(getJob().getLogger()).convert(renamedBam, cramFile, referenceGenome.getWorkingFastaFileGzipped(), true, threads, doArchival);
             }
             else
             {