Merge pull request #109 from LabKey/fb_merge_discvr-21.3

Merge discvr 21.3

Author: bbimber

.gitignore

@@ -5,9 +5,10 @@
 /travisSettings.sh
 
 # Created by npm install
-/jbrowse/jb_run.js
-/jbrowse/jb_setup.js
-/jbrowse/.gradle
+jbrowse/jb_run.js
+jbrowse/jb_setup.js
+jbrowse/.gradle
+jbrowse/resources/external/jb-cli/*
 
 # Ignore labkey-npm build artifacts
 **/gen/**

SequenceAnalysis/pipeline_code/sequence_tools_install.sh

@@ -276,6 +276,65 @@ else
     echo "Already installed"
 fi
 
+
+#
+# GATK3
+#
+echo ""
+echo ""
+echo "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%"
+echo "Install GATK3"
+echo ""
+cd $LKSRC_DIR
+
+if [[ ! -e ${LKTOOLS_DIR}/Queue.jar || ! -z $FORCE_REINSTALL ]];
+then
+    rm -Rf ${LKTOOLS_DIR}/Queue.jar
+    rm -Rf ${LKTOOLS_DIR}/GenomeAnalysisTK.jar
+    rm -Rf ${LKSRC_DIR}/gatk
+
+    mkdir -p gatk
+    cd gatk
+
+    echo "Downloading GATK from GIT"
+    git clone git://github.com/broadgsa/gatk-protected.git
+    cd gatk-protected
+    git checkout tags/3.7
+    cd ../
+
+    #this manually increases Xss in the hope of fixing intermittent StackOverlowErrors
+    sed -i '/<groupId>org.scala-tools<\/groupId>/a <configuration><jvmArgs><jvmArg>-Xss10m<\/jvmArg><\/jvmArgs><\/configuration>' ./gatk-protected/pom.xml
+
+    #fix multithreading bug
+    sed -i 's/private final List<GenomeLoc> upstreamDeletionsLoc = new LinkedList<>();/private final ThreadLocal< List<GenomeLoc> > upstreamDeletionsLoc = ThreadLocal.withInitial(() -> new LinkedList<GenomeLoc>());/g' ./gatk-protected/protected/gatk-tools-protected/src/main/java/org/broadinstitute/gatk/tools/walkers/genotyper/GenotypingEngine.java
+    sed -i 's/upstreamDeletionsLoc.add(genomeLoc);/upstreamDeletionsLoc.get().add(genomeLoc);/g' ./gatk-protected/protected/gatk-tools-protected/src/main/java/org/broadinstitute/gatk/tools/walkers/genotyper/GenotypingEngine.java
+    sed -i 's/upstreamDeletionsLoc.iterator();/upstreamDeletionsLoc.get().iterator();/g' ./gatk-protected/protected/gatk-tools-protected/src/main/java/org/broadinstitute/gatk/tools/walkers/genotyper/GenotypingEngine.java
+
+    #libVectorLogless compile:
+    #see: https://gatkforums.broadinstitute.org/gatk/discussion/9947/crashes-with-segmentation-fault-in-shipped-libvectorloglesspairhmm-so
+    sed -i 's/<!--<USE_GCC>1<\/USE_GCC>-->/<USE_GCC>1<\/USE_GCC>/g' ./gatk-protected/public/VectorPairHMM/pom.xml
+    sed -i 's/<!--<C_COMPILER>\/opt\/gcc-4.8.2\/bin\/gcc<\/C_COMPILER>-->/<C_COMPILER>\/usr\/bin\/gcc<\/C_COMPILER>/g' ./gatk-protected/public/VectorPairHMM/pom.xml
+    sed -i 's/<!--<CPP_COMPILER>\/opt\/gcc-4.8.2\/bin\/g++<\/CPP_COMPILER>-->/<CPP_COMPILER>\/usr\/bin\/g++<\/CPP_COMPILER>/g' ./gatk-protected/public/VectorPairHMM/pom.xml
+    cd gatk-protected/public/VectorPairHMM/
+    mvn verify
+    cd ../../../
+    cp gatk-protected/public/VectorPairHMM/target/libVectorLoglessPairHMM.so ${LKTOOLS_DIR}
+
+    cd gatk-protected
+
+    #remove due to compilation / dependency resolution error
+    rm ./public/external-example/src/main/java/org/mycompany/app/*
+    rm ./public/external-example/src/test/java/org/mycompany/app/*
+    sed -i '/<module>external-example<\/module>/d' ./public/pom.xml
+
+    #NOTE: can add -P\!queue to skip queue in some cases
+    mvn verify -U
+    mvn package
+    cp ./protected/gatk-package-distribution/target/gatk-package-distribution-3.7.jar ${LKTOOLS_DIR}/GenomeAnalysisTK.jar
+    cp ./protected/gatk-queue-package-distribution/target/gatk-queue-package-distribution-3.7.jar ${LKTOOLS_DIR}/Queue.jar
+fi
+
+
 if [[ ! -e ${LKTOOLS_DIR}/DISCVRSeq.jar || ! -z $FORCE_REINSTALL ]];
 then
     rm -Rf DISCVRSeq*

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/preprocessing/TagPcrSummaryStep.java

@@ -69,7 +69,7 @@ public static class Provider extends AbstractAnalysisStepProvider<TagPcrSummaryS
     {
         public Provider()
         {
-            super("Tag-PCR", "Tag-PCR Integration Sites", null, "This will produce a table summarizing unique alignments in this BAM.  It was originally created to summarize genomic insertions.", Arrays.asList(
+            super("Tag-PCR", "Map Integration Sites", null, "This will produce a table summarizing unique genome/transgene junctions using a BAM.", Arrays.asList(
                     ToolParameterDescriptor.create(OUTPUT_GENBANK, "Create Genbank Output", "If selected, this will output a genbank file summarizing amplicons and primers", "checkbox", new JSONObject(){{
                         put("checked", true);
                     }}, true),
@@ -349,7 +349,7 @@ public void execute(File bamFile, File referenceFasta, File outputTable, @Nullab
             List<String> args = new ArrayList<>();
             args.addAll(getBaseArgs());
 
-            args.add("TagPcrSummary");
+            args.add("IntegrationSiteMapper");
 
             args.add("--bam");
             args.add(bamFile.getPath());

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/AbstractGenomicsDBImportHandler.java

@@ -301,7 +301,7 @@ Set<String> getContigsInInputs(List<File> inputVCFs, Logger log) throws Pipeline
         return _contigsInInputs;
     }
 
-    private void copyToLevelFiles(PipelineJob job, File sourceWorkspace, File destinationWorkspace, boolean removeOtherFiles, boolean overwriteExisting) throws IOException
+    private void copyTopLevelFiles(PipelineJob job, File sourceWorkspace, File destinationWorkspace, boolean removeOtherFiles, boolean overwriteExisting) throws IOException, PipelineJobException
     {
         job.getLogger().info("Copying top-level files from: " + sourceWorkspace.getPath());
         if (removeOtherFiles)
@@ -333,6 +333,28 @@ private void copyToLevelFiles(PipelineJob job, File sourceWorkspace, File destin
 
             FileUtils.copyFile(source, dest);
         }
+
+        File metaDir = new File(sourceWorkspace, "genomicsdb_meta_dir");
+        File metaDirDest = new File(sourceWorkspace, "genomicsdb_meta_dir");
+        if (metaDirDest.exists())
+        {
+            if (!overwriteExisting)
+            {
+                job.getLogger().debug("workspace file exists, will not overwrite: " + metaDirDest.getPath());
+            }
+            else
+            {
+                FileUtils.deleteDirectory(metaDirDest);
+            }
+        }
+
+        if (metaDir.exists() && !metaDirDest.exists())
+        {
+            job.getLogger().debug("Copying directory with rsync: " + metaDir.getPath());
+            new SimpleScriptWrapper(job.getLogger()).execute(Arrays.asList(
+                    "rsync", "-r", "-a", "--delete", "--no-owner", "--no-group", "--no-perms", "--chmod=D2770,F660", metaDir.getPath(), metaDirDest.getParentFile().getPath()
+            ));
+        }
     }
 
     protected File getMarkerFile(File workspace)
@@ -684,7 +706,7 @@ private void copyWorkspace(JobContext ctx, File sourceWorkspace, File destinatio
 
                 if (!haveCopiedTopLevelFiles)
                 {
-                    copyToLevelFiles(ctx.getJob(), sourceWorkspace, destinationWorkspaceFolder, removeExistingTopLevelFiles, overwriteTopLevelFiles);
+                    copyTopLevelFiles(ctx.getJob(), sourceWorkspace, destinationWorkspaceFolder, removeExistingTopLevelFiles, overwriteTopLevelFiles);
                     haveCopiedTopLevelFiles = true;
                 }
 
@@ -707,7 +729,6 @@ private void copyWorkspace(JobContext ctx, File sourceWorkspace, File destinatio
                 }
                 else
                 {
-                    //TODO: dest permissions?
                     ctx.getLogger().debug("Copying directory with rsync: " + sourceFolder.getPath());
                     //NOTE: since neither path will end in slashes, rsync to the parent folder should result in the correct placement
                     new SimpleScriptWrapper(ctx.getLogger()).execute(Arrays.asList(

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/GenotypeGVCFsWrapper.java

@@ -226,7 +226,11 @@ else if (f.isDirectory() && doneFile.exists())
                 {
                     ctx.getLogger().info("Files will be marked for deletion after this step");
                     toDelete.add(movedFile);
-                    toDelete.add(movedIdx);
+                    if (movedIdx != null)
+                    {
+                        toDelete.add(movedIdx);
+                    }
+
                     if (doneFile.exists())
                     {
                         toDelete.add(doneFile);

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/MendelianViolationReportStep.java

@@ -57,13 +57,6 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
         VariantProcessingStepOutputImpl output = new VariantProcessingStepOutputImpl();
 
         List<String> options = new ArrayList<>();
-        Integer maxThreads = SequenceTaskHelper.getMaxThreads(getPipelineCtx().getLogger());
-        if (maxThreads != null)
-        {
-            options.add("-nt");
-            options.add(maxThreads.toString());
-        }
-
         if (getProvider().getParameterByName("excludeFiltered").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false))
         {
             options.add("--excludeFiltered");

SequenceAnalysis/test/src/org/labkey/test/tests/external/labModules/SequenceTest.java

@@ -1673,4 +1673,4 @@ private void basicOutputFilesTest(boolean sequencePipelineEnabled) throws Except
 
         Assert.assertFalse("File still exists: " + existing.getPath(), existing.exists());
     }
-}
+}

jbrowse/config/dev.config.js

@@ -27,5 +27,19 @@ clientConfig.plugins =
     [new webpack.ProvidePlugin({regeneratorRuntime: 'regenerator-runtime'}),
      new NodePolyfillPlugin()].concat(constants.processPlugins(entryPoints))
 
+clientConfig.module.rules = clientConfig.module.rules.concat(
+{
+    test: /\.m?js$/,
+    exclude: /node_modules/,
+    use: {
+        loader: 'babel-loader',
+        options: {
+            presets: [
+                ['@babel/preset-env', { targets: "defaults" }],
+                ['@babel/preset-react']
+            ]
+        }
+    }
+})
 
 module.exports = [clientConfig]

jbrowse/config/prod.config.js

@@ -27,5 +27,19 @@ clientConfig.plugins =
     [new webpack.ProvidePlugin({regeneratorRuntime: 'regenerator-runtime'}),
      new NodePolyfillPlugin()].concat(constants.processPlugins(entryPoints))
 
+clientConfig.module.rules = clientConfig.module.rules.concat(
+{
+    test: /\.m?js$/,
+    exclude: /node_modules/,
+    use: {
+        loader: 'babel-loader',
+        options: {
+            presets: [
+                ['@babel/preset-env', { targets: "defaults" }],
+                ['@babel/preset-react']
+            ]
+        }
+    }
+})
 
 module.exports = [clientConfig]

jbrowse/config/watch.config.js

@@ -27,5 +27,19 @@ clientConfig.plugins =
     [new webpack.ProvidePlugin({regeneratorRuntime: 'regenerator-runtime'}),
      new NodePolyfillPlugin()].concat(constants.processPlugins(entryPoints))
 
+clientConfig.module.rules = clientConfig.module.rules.concat(
+{
+    test: /\.m?js$/,
+    exclude: /node_modules/,
+    use: {
+        loader: 'babel-loader',
+        options: {
+            presets: [
+                ['@babel/preset-env', { targets: "defaults" }],
+                ['@babel/preset-react']
+            ]
+        }
+    }
+})
 
 module.exports = [clientConfig]