Merge pull request #19 from LabKey/fb_merge_discvr-19.3

Merge discvr-19.3 to develop, automatically created

Author: bbimber

.travis.yml

@@ -19,6 +19,7 @@ cache:
 
 install: skip
 script:
+  - wget -O ./travis.sh https://github.com/bimberlabinternal/DevOps/raw/master/travisci/travis.sh
   - bash ./travis.sh
 
 before_deploy:

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/SequenceAnalysisService.java

@@ -17,6 +17,7 @@
 
 
 import org.apache.log4j.Logger;
+import org.jetbrains.annotations.Nullable;
 import org.labkey.api.data.Container;
 import org.labkey.api.laboratory.NavItem;
 import org.labkey.api.pipeline.PipelineJob;
@@ -96,7 +97,7 @@ static public void setInstance(SequenceAnalysisService instance)
 
     abstract public String createReferenceLibrary(List<Integer> sequenceIds, Container c, User u, String name, String assemblyId, String description, boolean skipCacheIndexes, boolean skipTriggers, Set<GenomeTrigger> extraTriggers) throws IOException;
 
-    abstract public File combineVcfs(List<File> files, File outputGz, ReferenceGenome genome, Logger log) throws PipelineJobException;
+    abstract public File combineVcfs(List<File> files, File outputGz, ReferenceGenome genome, Logger log, boolean multiThreaded, @Nullable Integer compressionLevel) throws PipelineJobException;
 
     abstract public String getScriptPath(String moduleName, String path) throws PipelineJobException;
 }

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisServiceImpl.java

@@ -7,6 +7,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.SystemUtils;
 import org.apache.log4j.Logger;
+import org.jetbrains.annotations.Nullable;
 import org.labkey.api.data.CompareType;
 import org.labkey.api.data.Container;
 import org.labkey.api.data.ContainerManager;
@@ -473,8 +474,8 @@ public String createReferenceLibrary(List<Integer> sequenceIds, Container c, Use
     }
 
     @Override
-    public File combineVcfs(List<File> files, File outputGz, ReferenceGenome genome, Logger log) throws PipelineJobException
+    public File combineVcfs(List<File> files, File outputGz, ReferenceGenome genome, Logger log, boolean multiThreaded, @Nullable Integer compressionLevel) throws PipelineJobException
     {
-        return SequenceUtil.combineVcfs(files, genome, outputGz, log);
+        return SequenceUtil.combineVcfs(files, genome, outputGz, log, multiThreaded, compressionLevel);
     }
 }

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingRemoteMergeTask.java

@@ -1,6 +1,7 @@
 package org.labkey.sequenceanalysis.pipeline;
 
 import htsjdk.samtools.util.Interval;
+import org.apache.commons.lang3.StringUtils;
 import org.jetbrains.annotations.NotNull;
 import org.labkey.api.pipeline.AbstractTaskFactory;
 import org.labkey.api.pipeline.AbstractTaskFactorySettings;
@@ -138,13 +139,15 @@ private VariantProcessingJob getPipelineJob()
         }
         else
         {
-            if (getPipelineJob().getSequenceSupport().getCachedGenomes().size() != 1)
+            Set<Integer> genomeIds = new HashSet<>();
+            getPipelineJob().getFiles().forEach(x -> genomeIds.add(x.getLibrary_id()));
+            if (genomeIds.size() != 1)
             {
-                throw new PipelineJobException("Expected a single genome, found: " + getPipelineJob().getSequenceSupport().getCachedGenomes().size());
+                throw new PipelineJobException("Expected a single genome, found: " + StringUtils.join(genomeIds, ", "));
             }
 
-            ReferenceGenome genome = getPipelineJob().getSequenceSupport().getCachedGenomes().iterator().next();
-            combined = SequenceAnalysisService.get().combineVcfs(toConcat, combined, genome, getJob().getLogger());
+            ReferenceGenome genome = getPipelineJob().getSequenceSupport().getCachedGenome(genomeIds.iterator().next());
+            combined = SequenceAnalysisService.get().combineVcfs(toConcat, combined, genome, getJob().getLogger(), true, null);
         }
         manager.addOutput(action, "Merged VCF", combined);
 

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/BgzipRunner.java

@@ -16,6 +16,7 @@
 public class BgzipRunner extends AbstractCommandWrapper
 {
     private int _maxThreads = -1;
+    private Integer _compressionLevel = null;
 
     public BgzipRunner(@Nullable Logger logger)
     {
@@ -27,6 +28,11 @@ public void setMaxThreads(int maxThreads)
         _maxThreads = maxThreads;
     }
 
+    public void setCompressionLevel(int compressionLevel)
+    {
+        _compressionLevel = compressionLevel;
+    }
+
     public File execute(File input) throws PipelineJobException
     {
         getLogger().info("BGZipping file: " + input.getPath());
@@ -68,6 +74,12 @@ public List<String> getParams(File input)
             params.add(threads.toString());
         }
 
+        if (_compressionLevel != null)
+        {
+            params.add("--compress-level");
+            params.add(_compressionLevel.toString());
+        }
+
         params.add(input.getPath());
 
         return params;

SequenceAnalysis/src/org/labkey/sequenceanalysis/util/SequenceUtil.java

@@ -447,7 +447,7 @@ public static void sortROD(File input, Logger log, Integer startColumnIdx) throw
         sorted.delete();
     }
 
-    public static File combineVcfs(List<File> files, ReferenceGenome genome, File outputGzip, Logger log) throws PipelineJobException
+    public static File combineVcfs(List<File> files, ReferenceGenome genome, File outputGzip, Logger log, boolean multiThreaded, @Nullable Integer compressionLevel) throws PipelineJobException
     {
         log.info("combining VCFs: ");
 
@@ -500,13 +500,13 @@ else if (!samples.equals(header.getSampleNamesInOrder()))
                 writer.write("{\n");
                 bashCommands.forEach(x -> writer.write(x + '\n'));
 
-                Integer threads = SequencePipelineService.get().getMaxThreads(log);
+                Integer threads = multiThreaded ? SequencePipelineService.get().getMaxThreads(log) : null;
                 if (threads != null)
                 {
                     threads = Math.max(1, threads - 1);
                 }
 
-                writer.write("} | bgzip -f" + (threads == null ? "" : " --threads " + threads) + " > " + outputGzip + "\n");
+                writer.write("} | bgzip -f" + (compressionLevel == null ? "" : " --compress-level 9") + (threads == null ? "" : " --threads " + threads) + " > " + outputGzip + "\n");
             }
 
             SimpleScriptWrapper wrapper = new SimpleScriptWrapper(log);