Changes required to update HTSJDK Version (#34)

* Changes required to update HTSJDK Version
* Update jars.txt for new version
* Remove old code that attempts to find JARs directly

Author: bbimber

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/SequenceAnalysisService.java

@@ -89,8 +89,6 @@ static public void setInstance(SequenceAnalysisService instance)
 
     abstract public String getVCFLineCount(File vcf, Logger log, boolean passOnly) throws PipelineJobException;
 
-    abstract public File getPicardJar();
-
     abstract public File writeAllCellHashingBarcodes(File webserverDir, User u, Container c) throws PipelineJobException;
 
     abstract public String createReferenceLibrary(List<Integer> sequenceIds, Container c, User u, String name, String assemblyId, String description, boolean skipCacheIndexes, boolean skipTriggers) throws IOException;

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/PicardWrapper.java

@@ -6,7 +6,6 @@
 import org.jetbrains.annotations.Nullable;
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.pipeline.PipelineJobService;
-import org.labkey.api.sequenceanalysis.SequenceAnalysisService;
 import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
 
 import java.io.File;
@@ -63,16 +62,7 @@ public static File getPicardJar()
             path = PipelineJobService.get().getAppProperties().getToolsDirectory();
         }
 
-        File ret = path == null ? null : new File(path, "picard.jar");
-
-        // we only use the provided JAR if there is not one in the pipeline directory.  this allows the admin to update this JAR separate from the LK version
-        // however, if not otherwise provided, use the included one for convenience.
-        if (ret == null || !ret.exists())
-        {
-            return SequenceAnalysisService.get().getPicardJar();
-        }
-
-        return ret;
+        return path == null ? null : new File(path, "picard.jar");
     }
 
     protected File getJar()

SequenceAnalysis/build.gradle

@@ -26,8 +26,9 @@ dependencies {
    external "org.apache.commons:commons-math3:${commonsMath3Version}"
    // picard brings in a version of servlet-api and a very old one at that, so we excluded it
    // Note: if changing this, we might need to match the htsjdk version set in gradle.properties
-   external("com.github.broadinstitute:picard:2.18.4") {
+   external("com.github.broadinstitute:picard:2.22.4") {
       exclude group: "javax.servlet", module: "servlet-api"
+      exclude group: "org.apache.commons", module: "commons-collections4"
    }
 
    BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "published", depExtension: "module")

SequenceAnalysis/resources/credits/dependencies.txt

@@ -1,8 +1,8 @@
 # direct external dependencies for project :server:modules:DiscvrLabKeyModules:SequenceAnalysis
-htsjdk-2.14.3.jar
+htsjdk-2.21.3.jar
 biojava3-core-3.0.7.jar
 biojava3-genome-3.0.7.jar
 bzip2-0.9.1.jar
 sam-1.96.jar
 commons-math3-3.6.1.jar
-picard-2.18.4.jar
+picard-2.22.4.jar

SequenceAnalysis/resources/credits/jars.txt

@@ -5,6 +5,6 @@ sam-1.96.jar|picard tools|1.96|{link:picard tools|http://sourceforge.net/project
 biojava3-core-3.0.7.jar|biojava3-core|3.0.7|{link:biojava|http://biojava.org/wiki/Main_Page}|{link:GNU Lesser GPL V2.1|http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html}|bbimber|Java framework for processing biological data
 biojava3-genome-3.0.7.jar|biojava3-genome|3.0.7|{link:biojava|http://biojava.org/wiki/Main_Page}|{link:GNU Lesser GPL V2.1|http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html}|bbimber|Java framework for processing biological data
 commons-math3-3.6.1.jar|commons-math|3-3.6.1|{link:commons-math|http://commons.apache.org/proper/commons-math/}|{link:Apache License, Version 2.0|http://www.apache.org/licenses/LICENSE-2.0.txt}|bbimber|Self-contained mathematics and statistics components
-htsjdk-2.14.3.jar|htsjdk|2.14.3|{link:htsjdk|http://samtools.github.io/htsjdk/}|{link:MIT License|http://opensource.org/licenses/MIT}|bbimber|Description	A Java API for high-throughput sequencing data (HTS) formats
-picard-2.18.4.jar|Picard Tools Lib|2.18.4|{link:PicardTools|https://github.com/broadinstitute/picard}|{link:Apache 2.0|http://www.apache.org/licenses/LICENSE-2.0}|bbimber|Manipulating Sequence Data
+htsjdk-2.21.3.jar|htsjdk|2.21.3|{link:htsjdk|http://samtools.github.io/htsjdk/}|{link:MIT License|http://opensource.org/licenses/MIT}|bbimber|Description	A Java API for high-throughput sequencing data (HTS) formats
+picard-2.22.4.jar|Picard Tools Lib|2.22.4|{link:PicardTools|https://github.com/broadinstitute/picard}|{link:Apache 2.0|http://www.apache.org/licenses/LICENSE-2.0}|bbimber|Manipulating Sequence Data
 {table}

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisManager.java

@@ -58,8 +58,6 @@
 import org.labkey.api.query.UserSchema;
 import org.labkey.api.reader.FastaDataLoader;
 import org.labkey.api.reader.FastaLoader;
-import org.labkey.api.resource.FileResource;
-import org.labkey.api.resource.Resource;
 import org.labkey.api.security.User;
 import org.labkey.api.security.UserManager;
 import org.labkey.api.security.permissions.DeletePermission;
@@ -70,7 +68,6 @@
 import org.labkey.api.util.Job;
 import org.labkey.api.util.JobRunner;
 import org.labkey.api.util.PageFlowUtil;
-import org.labkey.api.util.Path;
 import org.labkey.api.view.UnauthorizedException;
 import org.labkey.sequenceanalysis.model.ReferenceLibraryMember;
 import org.labkey.sequenceanalysis.pipeline.ReferenceGenomeImpl;
@@ -522,38 +519,6 @@ public void deleteRefAaSequence(List<Integer> rowIds) throws SQLException
         }
     }
 
-    private static final String htsjdkVersion = "2.14.3";
-    private static final String picardVersion = "2.18.4";
-
-    public static File getHtsJdkJar()
-    {
-        return getJar("htsjdk-" + htsjdkVersion);
-    }
-
-    public static File getPicardJar()
-    {
-        return getJar("picard-" + picardVersion);
-    }
-
-    private static File getJar(String name)
-    {
-        Resource r = ModuleLoader.getInstance().getResource(ModuleLoader.getInstance().getModule(SequenceAnalysisModule.NAME), Path.parse("lib/" + name + ".jar"));
-        if (r == null)
-        {
-            throw new IllegalArgumentException("Unable to find JAR file: " + name);
-        }
-        else if (!(r instanceof FileResource))
-        {
-            throw new IllegalArgumentException("resource not instance of FileResource: " + r.getClass().getName());
-        }
-
-        File jar = ((FileResource) r).getFile();
-        if (!jar.exists())
-            throw new RuntimeException("Not found: " + jar.getPath());
-
-        return jar;
-    }
-
     public static void cascadeDelete(int userId, String containerId, String schemaName, String queryName, String keyField, Object keyValue) throws SQLException
     {
         cascadeDelete(userId, containerId, schemaName, queryName, keyField, keyValue, null);
@@ -905,14 +870,6 @@ public static class TestCase extends Assert
         @Test
         public void testJars()
         {
-            File htsjdkJar = SequenceAnalysisManager.getHtsJdkJar();
-            assertNotNull("Unble to find HTSJDK jar", htsjdkJar);
-            assertTrue("JAR does not exist: " + htsjdkJar.getPath(), htsjdkJar.exists());
-
-            File picardJar = SequenceAnalysisManager.getPicardJar();
-            assertNotNull("Unable to find picard jar", picardJar);
-            assertTrue("JAR does not exist: " + picardJar.getPath(), picardJar.exists());
-
             File libDir = new File(ModuleLoader.getInstance().getModule(SequenceAnalysisModule.NAME).getExplodedPath(), "lib");
             assertNotNull("Unable to find SequenceAnalysis lib dir", libDir);
             assertTrue("SequenceAnalysis lib dir does not exist: " + libDir.getPath(), libDir.exists());

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisServiceImpl.java

@@ -432,12 +432,6 @@ public String getVCFLineCount(File vcf, Logger log, boolean passOnly) throws Pip
         return ProcessVariantsHandler.getVCFLineCount(vcf, log, passOnly);
     }
 
-    @Override
-    public File getPicardJar()
-    {
-        return SequenceAnalysisManager.getPicardJar();
-    }
-
     @Override
     public String getScriptPath(String moduleName, String path) throws PipelineJobException
     {

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/PicardMetricsUtil.java

@@ -9,9 +9,9 @@
 import org.labkey.api.data.ConvertHelper;
 import org.labkey.api.pipeline.PipelineJobException;
 import picard.analysis.AlignmentSummaryMetrics;
-import picard.analysis.CollectWgsMetrics;
 import picard.analysis.CollectWgsMetricsWithNonZeroCoverage;
 import picard.analysis.InsertSizeMetrics;
+import picard.analysis.WgsMetrics;
 import picard.sam.DuplicationMetrics;
 
 import java.io.BufferedReader;
@@ -56,7 +56,7 @@ else if (metrics.get(0).getClass() == AlignmentSummaryMetrics.class)
                 log.info("Importing Picard AlignmentSummaryMetricsCollector metrics from: " + f.getName());
                 return processAlignmentSummaryMetrics(metricsFile, log);
             }
-            else if (metrics.get(0).getClass() == CollectWgsMetrics.WgsMetrics.class)
+            else if (metrics.get(0).getClass() == WgsMetrics.class)
             {
                 log.info("Importing Picard WgsMetrics for: " + f.getName());
                 return processWgsMetrics(metricsFile, log);
@@ -92,9 +92,9 @@ private static List<Map<String, Object>> processWgsNonZeroMetrics(MetricsFile mf
     private static List<Map<String, Object>> processWgsMetricsBase(MetricsFile mf, Logger log, String category) throws PipelineJobException
     {
         List<Map<String, Object>> ret = new ArrayList<>();
-        List<CollectWgsMetrics.WgsMetrics> metrics = mf.getMetrics();
+        List<WgsMetrics> metrics = mf.getMetrics();
 
-        for (CollectWgsMetrics.WgsMetrics m : metrics)
+        for (WgsMetrics m : metrics)
         {
             Map<String, Object> metricNames = new HashMap<>();
 

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/alignment/MosaikWrapper.java

@@ -30,7 +30,6 @@
 import org.labkey.api.util.FileUtil;
 import org.labkey.api.util.PageFlowUtil;
 import org.labkey.sequenceanalysis.pipeline.SequenceTaskHelper;
-import org.labkey.sequenceanalysis.run.util.FixBAMWrapper;
 import org.labkey.sequenceanalysis.util.SequenceUtil;
 
 import java.io.File;
@@ -388,8 +387,8 @@ public File executeMosaikAligner(File refFasta, File indexDir, File reads, File
             }
 
             //Note: mosaik has started to give errors about bin field
-            FixBAMWrapper fixBam = new FixBAMWrapper(getLogger());
-            fixBam.executeCommand(bam, null);
+            //FixBAMWrapper fixBam = new FixBAMWrapper(getLogger());
+            //fixBam.executeCommand(bam, null);
 
             new SamSorter(getLogger()).execute(bam, null, SAMFileHeader.SortOrder.coordinate);