Additional support for VCF indexing

Author: bbimber

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractDiscvrSeqWrapper.java

@@ -10,6 +10,12 @@ public AbstractDiscvrSeqWrapper(Logger log)
         super(log);
     }
 
+    @Override
+    protected String getPackageName()
+    {
+        return "DISCVRSEQPATH";
+    }
+
     @Override
     protected String getJarName()
     {

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatk4Wrapper.java

@@ -3,7 +3,7 @@
 import htsjdk.variant.utils.SAMSequenceDictionaryExtractor;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.LogManager;
+import org.jetbrains.annotations.Nullable;
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.pipeline.PipelineJobService;
 import org.labkey.api.sequenceanalysis.SequenceAnalysisService;
@@ -36,7 +36,7 @@ protected String getJarName()
 
     protected File getJAR()
     {
-        String path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath("GATKPATH");
+        String path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath(getPackageName());
         if (path != null)
         {
             return new File(path);
@@ -56,7 +56,7 @@ public void setMaxRamOverride(Integer maxRamOverride)
         _maxRamOverride = maxRamOverride;
     }
 
-    protected void addJavaHomeToEnvironment()
+    public void addJava8HomeToEnvironment()
     {
         //since GATK requires java8, set JAVA_HOME to match this:
         File java8 = new File(SequencePipelineService.get().getJava8FilePath()).getParentFile();
@@ -98,13 +98,28 @@ public String getVersionString() throws PipelineJobException
     }
 
     public List<String> getBaseArgs()
+    {
+        return getBaseArgs(null);
+    }
+
+    protected String getPackageName()
+    {
+        return "GATKPATH";
+    }
+
+    public List<String> getBaseArgs(@Nullable String toolName)
     {
         List<String> args = new ArrayList<>();
         args.add(SequencePipelineService.get().getJava8FilePath());
         args.addAll(SequencePipelineService.get().getJavaOpts(_maxRamOverride));
         args.add("-jar");
         args.add(getJAR().getPath());
 
+        if (toolName != null)
+        {
+            args.add(toolName);
+        }
+
         return args;
     }
 

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/DISCVRSeqRunner.java

@@ -1,53 +1,11 @@
 package org.labkey.api.sequenceanalysis.run;
 
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.LogManager;
-import org.labkey.api.pipeline.PipelineJobService;
-import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
-import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper;
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
 
-public class DISCVRSeqRunner extends AbstractCommandWrapper
+public class DISCVRSeqRunner extends AbstractDiscvrSeqWrapper
 {
     public DISCVRSeqRunner(Logger log)
     {
         super(log);
     }
-
-    protected String getJarName()
-    {
-        return "DISCVRSeq.jar";
-    }
-
-    protected File getJar()
-    {
-        String path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath("DISCVRSEQPATH");
-        if (path != null)
-        {
-            return new File(path);
-        }
-
-        path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath(SequencePipelineService.SEQUENCE_TOOLS_PARAM);
-        if (path == null)
-        {
-            path = PipelineJobService.get().getAppProperties().getToolsDirectory();
-        }
-
-        return path == null ? new File(getJarName()) : new File(path, getJarName());
-
-    }
-
-    public List<String> getBaseArgs(String toolName)
-    {
-        List<String> args = new ArrayList<>();
-        args.add(SequencePipelineService.get().getJava8FilePath());
-        args.addAll(SequencePipelineService.get().getJavaOpts());
-        args.add("-jar");
-        args.add(getJar().getPath());
-        args.add(toolName);
-
-        return args;
-    }
 }

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisServiceImpl.java

@@ -275,7 +275,7 @@ public File ensureVcfIndex(File vcf, Logger log, boolean forceRecreate) throws I
                 }
                 else
                 {
-                    Index idx = IndexFactory.createDynamicIndex(vcf, new VCFCodec());
+                    Index idx = vcf.getName().toLowerCase().endsWith(".gz") ? IndexFactory.createIndex(vcf, new VCFCodec(), IndexFactory.IndexType.TABIX) : IndexFactory.createDynamicIndex(vcf, new VCFCodec());
                     idx.writeBasedOnFeatureFile(vcf);
                     if (!expectedIdx.exists())
                     {

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java

@@ -272,6 +272,11 @@ public String getJava8FilePath()
             java8Home = StringUtils.trimToNull(System.getenv("JDK_18"));
         }
 
+        if (java8Home == null && StringUtils.trimToNull(System.getenv("JAVA_HOME_8")) != null)
+        {
+            java8Home = StringUtils.trimToNull(System.getenv("JAVA_HOME_8"));
+        }
+
         if (java8Home != null)
         {
             File ret = new File(java8Home, "bin");

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/PbsvJointCallingHandler.java

@@ -230,6 +230,11 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
 
         private File runPbsvCall(JobContext ctx, List<File> inputs, ReferenceGenome genome, String outputBaseName, @Nullable String contig, boolean jobCompleted) throws PipelineJobException
         {
+            if (contig != null)
+            {
+                ctx.getJob().setStatus(PipelineJob.TaskStatus.running, "Processing: " + contig);
+            }
+
             if (inputs.isEmpty())
             {
                 throw new PipelineJobException("No inputs provided");

jbrowse/src/org/labkey/jbrowse/model/JsonFile.java

@@ -856,7 +856,7 @@ public File prepareResource(Logger log, boolean throwIfNotPrepared, boolean forc
 
         // Ensure index check runs even if file was already gzipped:
         File idx = new File(targetFile.getPath() + ".tbi");
-        if (needsGzip() && (forceReprocess || !idx.exists()))
+        if (targetFile.getName().toLowerCase().endsWith(".gz") && (forceReprocess || !idx.exists()))
         {
             createIndex(targetFile, log, idx, throwIfNotPrepared);
         }
@@ -935,11 +935,13 @@ public File prepareResource(Logger log, boolean throwIfNotPrepared, boolean forc
         return targetFile;
     }
 
-    private void prepareLuceneIndex(Logger log)
+    private void prepareLuceneIndex(Logger log) throws PipelineJobException
     {
         log.debug("Generating VCF full text index for file: " + getExpData().getFile().getName());
 
         DISCVRSeqRunner runner = new DISCVRSeqRunner(log);
+        runner.addJava8HomeToEnvironment();
+
         List<String> args = runner.getBaseArgs("VcfToLuceneIndexer");
         args.add("-V");
         args.add(getExpData().getFile().getPath());
@@ -977,6 +979,8 @@ private void prepareLuceneIndex(Logger log)
                 args.add(field);
             }
         }
+
+        runner.execute(args);
     }
 
     protected void createIndex(File finalLocation, Logger log, File idx, boolean throwIfNotPrepared) throws PipelineJobException
@@ -1001,6 +1005,10 @@ else if (TRACK_TYPES.gtf.getFileType().isType(finalLocation) || TRACK_TYPES.gff.
                         Index index = IndexFactory.createIndex(finalLocation.toPath(), new Gff3Codec(), IndexFactory.IndexType.TABIX);
                         index.write(idx);
                     }
+                    else if (TRACK_TYPES.vcf.getFileType().isType(finalLocation))
+                    {
+                        SequenceAnalysisService.get().ensureVcfIndex(finalLocation, log);
+                    }
                     else
                     {
                         log.warn("Cannot create tabix index on windows!");