Merge pull request #253 from LabKey/fb_merge_23.11_to_develop

Merge discvr-23.11 to develop

Author: bbimber

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/SequenceOutputFile.java

@@ -17,6 +17,8 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import org.json.JSONObject;
+import org.labkey.api.data.Container;
+import org.labkey.api.data.ContainerManager;
 import org.labkey.api.data.DbSchema;
 import org.labkey.api.data.TableSelector;
 import org.labkey.api.exp.api.ExpData;
@@ -159,6 +161,11 @@ public String getContainer()
         return _container;
     }
 
+    public Container getContainerObj()
+    {
+        return ContainerManager.getForId(_container);
+    }
+
     public void setContainer(String container)
     {
         _container = container;

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/ReferenceGenomeManager.java

@@ -105,7 +105,7 @@ public void cacheGenomeLocally(ReferenceGenome genome, Logger log) throws Pipeli
         File localCacheDir = SequencePipelineService.get().getRemoteGenomeCacheDirectory();
         if (isUpToDate(genome))
         {
-            log.debug("Genome up-to-date, will not repeat rsync");
+            log.debug("Genome up-to-date, will not repeat rsync: " + genome.getGenomeId());
             genome.setWorkingFasta(new File(new File(localCacheDir, genome.getGenomeId().toString()), genome.getSourceFastaFile().getName()));
 
             return;

SequenceAnalysis/pipeline_code/sequence_tools_install.sh

@@ -510,17 +510,19 @@ then
     rm -Rf bcftools*
     rm -Rf $LKTOOLS_DIR/bcftools
 
-    wget $WGET_OPTS https://github.com/samtools/bcftools/releases/download/1.16/bcftools-1.16.tar.bz2
-    bunzip2 bcftools-1.16.tar.bz2
-    tar -xf bcftools-1.16.tar
-    echo "Compressing TAR"
-    bzip2 bcftools-1.16.tar
-    chmod 755 bcftools-1.16
-    cd bcftools-1.16
-    ./configure
+    wget $WGET_OPTS https://github.com/samtools/bcftools/releases/download/1.18/bcftools-1.18.tar.bz2
+    tar xjvf bcftools-1.18.tar.bz2
+    bzip2 bcftools-1.18.tar
+    chmod 755 bcftools-1.18
+    cd bcftools-1.18
+    rm -f plugins/liftover.c
+    wget $WGET_OPTS -P plugins https://raw.githubusercontent.com/freeseek/score/master/liftover.c
+
+    ./configure CFLAGS="-g -Wall -O2 -std=c99"
     make
 
     install ./bcftools $LKTOOLS_DIR
+    install ./plugins/liftover.so $LKTOOLS_DIR
 else
     echo "Already installed"
 fi

SequenceAnalysis/resources/queries/pipeline/Job/Sequence Jobs Without Data.qview.xml

@@ -9,6 +9,7 @@
     </columns>
     <filters>
         <filter column="sequenceJobWithoutData" operator="eq" value="true"/>
+        <filter column="JobParent" operator="isblank" value=""/>
     </filters>
     <sorts>
         <sort column="Created" descending="true"/>

SequenceAnalysis/resources/web/SequenceAnalysis/field/SequenceOutputFileSelectorField.js

@@ -4,14 +4,14 @@ Ext4.define('SequenceAnalysis.field.SequenceOutputFileSelectorField', {
 
     genomeId: -1,
     performGenomeFilter: true,
+    valueField: 'dataid',
 
     initComponent: function(){
         Ext4.apply(this, {
             forceSelection: true,
             displayField: 'name',
-            valueField: 'dataid',
             listConfig: {
-                innerTpl: ['{name} ({[values["rowid"]]})']
+                innerTpl: ['{name} ({[values["' + this.valueField + '"]]})']
             },
             store: {
                 type: 'labkey-store',
@@ -21,7 +21,7 @@ Ext4.define('SequenceAnalysis.field.SequenceOutputFileSelectorField', {
                 autoLoad: true,
                 filterArray: this.getFilterArray(),
                 sort: 'name',
-                columns: 'library_id,name,dataid,category'
+                columns: 'rowid,library_id,name,dataid,category'
             }
         });
 

SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js

@@ -268,6 +268,12 @@ Ext4.define('SequenceAnalysis.panel.VariantProcessingPanel', {
 				label: 'Intervals to Exclude',
 				description: 'A BED or similar file with intervals to skip',
 				defaultValue: null
+			},{
+				fieldXtype: 'ldk-expdatafield',
+				name: 'skipExcessHetAndInbreeding',
+				label: 'Skip Excess Het And Inbreeding',
+				description: 'If checked, the ExcessHet and InbreedingCoeff annotations will be skipped, which can be important when using force-output-intervals',
+				defaultValue: null
 			},{
 				fieldXtype: 'checkbox',
 				name: 'doCopyInputs',

SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/GenotypeGVCFHandler.java

@@ -636,11 +636,14 @@ private void processOneInput(JobContext ctx, PipelineJob job, ReferenceGenome ge
             }
 
             // NOTE: added to side-step https://github.com/broadinstitute/gatk/issues/7938
-            toolParams.add("-AX");
-            toolParams.add("InbreedingCoeff");
+            if (ctx.getParams().optBoolean("variantCalling.GenotypeGVCFs.skipExcessHetAndInbreeding"))
+            {
+                toolParams.add("-AX");
+                toolParams.add("InbreedingCoeff");
 
-            toolParams.add("-AX");
-            toolParams.add("ExcessHet");
+                toolParams.add("-AX");
+                toolParams.add("ExcessHet");
+            }
 
             if (!ctx.getParams().isNull("variantCalling.GenotypeGVCFs.maxGenotypeCount"))
             {

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/VariantAnnotatorStep.java

@@ -49,6 +49,9 @@ public Provider()
                     }}, null),
                     ToolParameterDescriptor.create("impact", "IMPACT", "If selected, the Impact annotation will run, which parses the SnpEff ANN field into multiple discrete INFO fields.", "checkbox", new JSONObject(){{
                         put("checked", false);
+                    }}, null),
+                    ToolParameterDescriptor.create("excessHet", "ExcessHet", "If selected, the ExcessHet annotation will run.", "checkbox", new JSONObject(){{
+                        put("checked", false);
                     }}, null)
             ), null, "");
         }
@@ -106,6 +109,12 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
             options.add("Impact");
         }
 
+        if (getProvider().getParameterByName("excessHet").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false))
+        {
+            options.add("-A");
+            options.add("ExcessHet");
+        }
+
         if (intervals != null)
         {
             intervals.forEach(interval -> {

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/VariantQCStep.java

@@ -1,6 +1,7 @@
 package org.labkey.sequenceanalysis.run.variant;
 
 import htsjdk.samtools.util.Interval;
+import org.apache.commons.io.FileUtils;
 import org.json.JSONObject;
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.sequenceanalysis.pipeline.AbstractPipelineStep;
@@ -16,8 +17,8 @@
 
 import javax.annotation.Nullable;
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 
 /**
@@ -40,7 +41,10 @@ public Provider()
                     ToolParameterDescriptor.create("writeJson", "Write Raw Data", "If selected, both an HTML report and a text file with the raw data will be created.", "checkbox", new JSONObject()
                     {{
                         put("checked", true);
-                    }}, true)
+                    }}, true),
+                    ToolParameterDescriptor.create("doCopyLocal", "Copy Input To Working Directory", "If selected, the input VCF will always be copied to the working directory, if it is not already present.", "checkbox", new JSONObject(){{
+                        put("checked", false);
+                    }}, false)
             ), null, "https://bimberlab.github.io/DISCVRSeq/");
         }
 
@@ -56,6 +60,39 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
     {
         VariantProcessingStepOutputImpl output = new VariantProcessingStepOutputImpl();
 
+        boolean doCopyLocal = getProvider().getParameterByName("doCopyLocal").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), boolean.class, false);
+        if (doCopyLocal)
+        {
+            File local = new File(outputDirectory, inputVCF.getName());
+            File localIdx = new File(local.getPath() + ".tbi");
+            if (!inputVCF.equals(local))
+            {
+                getPipelineCtx().getLogger().debug("Making local copy of VCF:");
+                if (local.exists())
+                {
+                    local.delete();
+                }
+
+                if (localIdx.exists())
+                {
+                    localIdx.delete();
+                }
+
+                try
+                {
+                    FileUtils.copyFile(inputVCF, local);
+                    FileUtils.copyFile(new File(inputVCF.getPath() + ".tbi"), localIdx);
+                    output.addIntermediateFile(local);
+                    output.addIntermediateFile(localIdx);
+                    inputVCF = local;
+                }
+                catch (IOException e)
+                {
+                    throw new PipelineJobException(e);
+                }
+            }
+        }
+
         List<String> options = new ArrayList<>();
 
         File pedFile = ProcessVariantsHandler.getPedigreeFile(getPipelineCtx().getSourceDirectory(true));

jbrowse/api-src/org/labkey/api/jbrowse/JBrowseService.java

@@ -7,8 +7,11 @@
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.pipeline.PipelineValidationException;
 import org.labkey.api.security.User;
+import org.labkey.api.sequenceanalysis.SequenceOutputFile;
 
+import java.io.File;
 import java.io.IOException;
+import java.util.List;
 
 /**
  * Created by bimber on 11/3/2016.
@@ -38,4 +41,17 @@ static public void setInstance(JBrowseService instance)
     abstract public void registerGroupsProvider(GroupsProvider provider);
 
     abstract public void registerFieldCustomizer(JBrowseFieldCustomizer customizer);
+
+    abstract public void prepareLuceneIndex(File vcf, File indexDir, Logger log, List<String> infoFieldsForFullTextSearch, boolean allowLenientLuceneProcessing) throws PipelineJobException;
+
+    abstract public SequenceOutputFile findMatchingLuceneIndex(SequenceOutputFile vcfFile, List<String> infoFieldsToIndex, User u, @Nullable Logger log) throws PipelineJobException;
+
+    abstract public void registerLuceneIndexDetector(LuceneIndexDetector detector);
+
+    public interface LuceneIndexDetector
+    {
+        SequenceOutputFile findMatchingLuceneIndex(SequenceOutputFile vcfFile, List<String> infoFieldsToIndex, User u, @Nullable Logger log) throws PipelineJobException;
+
+        boolean isAvailable(Container c);
+    }
 }