Merge discvr-20.3 to develop

Author: bbimber

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/SequenceAnalysisService.java

@@ -100,4 +100,6 @@ static public void setInstance(SequenceAnalysisService instance)
     abstract public File combineVcfs(List<File> files, File outputGz, ReferenceGenome genome, Logger log, boolean multiThreaded, @Nullable Integer compressionLevel) throws PipelineJobException;
 
     abstract public String getScriptPath(String moduleName, String path) throws PipelineJobException;
+
+    abstract public void sortGxf(Logger log, File input, @Nullable File output) throws PipelineJobException;
 }

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AnalysisStep.java

@@ -20,7 +20,6 @@
 import org.labkey.api.sequenceanalysis.model.Readset;
 
 import java.io.File;
-import java.util.List;
 
 /**
  * User: bimber
@@ -41,11 +40,13 @@ default void init(SequenceAnalysisJobSupport support) throws PipelineJobExceptio
 
     /**
      * Will perform analysis steps on the remote pipeline server
+     * Note: outputDir is the location where the BAM is expected to be written
      */
     public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, ReferenceGenome referenceGenome, File outputDir) throws PipelineJobException;
 
     /**
      * Will perform analysis steps on the local webserver
+     * Note: outputDir is the location where the BAM was created (if this step creates a BAM)
      */
     public Output performAnalysisPerSampleLocal(AnalysisModel model, File inputBam, File referenceFasta, File outDir) throws PipelineJobException;
 

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequenceOutputHandler.java

@@ -245,8 +245,13 @@ public interface HasActionNames
 
     public static interface TracksVCF
     {
-        public File getFinalVCF(JobContext ctx) throws PipelineJobException;
+        public File getScatterJobOutput(JobContext ctx) throws PipelineJobException;
 
-        public SequenceOutputFile createFinalSequenceOutput(PipelineJob job, File processed, List<SequenceOutputFile> inputFiles);
+        public SequenceOutputFile createFinalSequenceOutput(PipelineJob job, File processed, List<SequenceOutputFile> inputFiles) throws PipelineJobException;
+    }
+
+    public static interface HasCustomVariantMerge
+    {
+        public File performVariantMerge(TaskFileManager manager, RecordedAction action, SequenceOutputHandler<SequenceOutputHandler.SequenceOutputProcessor> handler, PipelineJob job) throws PipelineJobException;
     }
 }

SequenceAnalysis/pipeline_code/sequence_tools_install.sh

@@ -175,17 +175,15 @@ cd $LKSRC_DIR
 if [[ ! -e ${LKTOOLS_DIR}/bwa || ! -z $FORCE_REINSTALL ]];
 then
     echo "Cleaning up previous installs"
-    rm -Rf bwa-0.6.2*
-    rm -Rf bwa-0.7.9a*
-    rm -Rf bwa-0.7.12*
+    rm -Rf bwa-0.*
     rm -Rf $LKTOOLS_DIR/bwa
 
-    wget $WGET_OPTS https://downloads.sourceforge.net/project/bio-bwa/bwa-0.7.12.tar.bz2
-    bunzip2 bwa-0.7.12.tar.bz2
-    tar -xf bwa-0.7.12.tar
-    bzip2 bwa-0.7.12.tar
-    cd bwa-0.7.12
-    make CFLAGS=-msse2
+    wget $WGET_OPTS https://downloads.sourceforge.net/project/bio-bwa/bwa-0.7.17.tar.bz2
+    bunzip2 bwa-0.7.17.tar.bz2
+    tar -xf bwa-0.7.17.tar
+    bzip2 bwa-0.7.17.tar
+    cd bwa-0.7.17
+    make
     install bwa $LKTOOLS_DIR/
 else
     echo "Already installed"

SequenceAnalysis/resources/schemas/dbscripts/postgresql/SequenceAnalysis-12.320-12.321.sql

@@ -0,0 +1 @@
+ALTER TABLE sequenceanalysis.chain_files ADD source varchar(4000);

SequenceAnalysis/resources/schemas/dbscripts/sqlserver/SequenceAnalysis-12.320-12.321.sql

@@ -0,0 +1 @@
+ALTER TABLE sequenceanalysis.chain_files ADD source varchar(4000);

SequenceAnalysis/resources/schemas/sequenceanalysis.xml

@@ -2607,6 +2607,9 @@
                     <fkTable>data</fkTable>
                     <fkColumnName>RowId</fkColumnName>
                 </fk>
+            </column>
+            <column columnName="source">
+
             </column>
             <column columnName="version">
 

SequenceAnalysis/resources/web/SequenceAnalysis/field/SequenceOutputFileSelectorField.js

@@ -0,0 +1,83 @@
+Ext4.define('SequenceAnalysis.field.SequenceOutputFileSelectorField', {
+    extend: 'LABKEY.ext4.ComboBox',
+    alias: 'widget.sequenceanalysis-sequenceoutputfileselectorfield',
+
+    genomeId: -1,
+
+    initComponent: function(){
+        Ext4.apply(this, {
+            forceSelection: true,
+            displayField: 'name',
+            valueField: 'dataid',
+            listConfig: {
+                innerTpl: ['{name} ({[values["rowid"]]})']
+            },
+            store: {
+                type: 'labkey-store',
+                containerPath: Laboratory.Utils.getQueryContainerPath(),
+                schemaName: 'sequenceanalysis',
+                queryName: 'outputfiles',
+                autoLoad: true,
+                filterArray: this.getFilterArray(),
+                sort: 'name',
+                columns: 'library_id,name,dataid,category'
+            }
+        });
+
+        this.callParent(arguments);
+
+        this.on('afterrender', function() {
+            var parent = this.up('sequenceanalysis-basesequencepanel');  //Alignment panels
+            var window = this.up('window'); //OutputHandlerWindow
+            if (parent) {
+                var field = parent.down('field[name=referenceLibraryCreation.SavedLibrary.libraryId]');
+                if (field) {
+                    this.mon(field, 'change', this.onGenomeChange, this);
+                    if (field.getValue()){
+                        this.updateStoreFilters(field.getValue());
+                    }
+                }
+                else if (parent.libraryIds){
+                    if (parent.libraryIds.length === 1){
+                        this.updateStoreFilters(parent.libraryIds[0]);
+                    }
+                    else if (!parent.libraryIds.length){
+                        Ext4.Msg.alert('Genome Error', 'There are no reference genomes associated with these samples');
+                    }
+                    else if (parent.libraryIds.length > 1){
+                        Ext4.Msg.alert('Genome Error', 'More than one reference genome associated with these samples.  They must all use the same genome.');
+                    }
+                }
+                else {
+                    LDK.Utils.logError('unable to find library field in GenomeFileSelectorField');
+                }
+            }
+            else if (window && window.libraryId){
+                this.updateStoreFilters(window.libraryId);
+            }
+            else {
+                LDK.Utils.logError('unable to find basesequencepanel in SequenceOutputFileSelectorField');
+                Ext4.Msg.alert('Error', 'There is no genome ID provided to this field.  This indicates an error in how the module was developed - please contact your administrator.');
+            }
+        }, this);
+    },
+
+    onGenomeChange: function(field, genomeId){
+        genomeId = genomeId ? genomeId : -1;
+        if (genomeId === this.genomeId){
+            return;
+        }
+
+        this.updateStoreFilters(genomeId);
+    },
+
+    updateStoreFilters: function(genomeId){
+        this.genomeId = genomeId;
+        this.store.filterArray = this.getFilterArray();
+        this.store.load();
+    },
+
+    getFilterArray: function(){
+        return [LABKEY.Filter.create('library_id', this.genomeId), LABKEY.Filter.create('category', this.category, LABKEY.Filter.Types.EQUAL)];
+    }
+});

SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantScatterGatherPanel.js

@@ -17,7 +17,7 @@ Ext4.define('SequenceAnalysis.panel.VariantScatterGatherPanel', {
                 labelWidth: this.labelWidth,
                 width: this.defaultFieldWidth,
                 expandToFitContent: true,
-                value: 'none',
+                value: this.defaultValue || 'none',
                 displayField: 'label',
                 valueField: 'value',
                 store: {
@@ -36,6 +36,9 @@ Ext4.define('SequenceAnalysis.panel.VariantScatterGatherPanel', {
                 allowBlank: false,
                 listeners: {
                     scope: this,
+                    render: function(field){
+                        field.fireEvent('change', field, field.getValue());
+                    },
                     change: function(field, val) {
                         var panel = this.down('#scatterGatherMethodOptions');
                         panel.removeAll();

SequenceAnalysis/resources/web/SequenceAnalysis/window/ImportChainFileWindow.js

@@ -20,7 +20,7 @@ Ext4.define('SequenceAnalysis.window.ImportChainFileWindow', {
                 width: 450
             },
             items: [{
-                html: 'This will import a chain file, which is used to liftover variants or annotations from one reference genome to another.',
+                html: 'This will import a chain file, which is used to liftover variants or annotations from one reference genome to another. Note: because UCSC is a common source of chain files and they use non-standard contig names, the server-side code will attempt to parse any non-standard names in the chain file, match them with the genomes, and write an updated chain file.',
                 style: 'padding-bottom: 10px;',
                 width: null
             },{
@@ -58,11 +58,23 @@ Ext4.define('SequenceAnalysis.window.ImportChainFileWindow', {
                     fieldLabel: 'File',
                     name: 'chainFile',
                     allowBlank: false
+                },{
+                    xtype: 'textfield',
+                    fieldLabel: 'Source',
+                    name: 'source',
+                    allowBlank: false
                 },{
                     xtype: 'ldk-numberfield',
                     fieldLabel: 'Version',
+                    helpPopup: 'This field is used by the system - when a chain file is being selected automatically it will choose the one with the highest version',
                     name: 'version',
                     allowBlank: false
+                },{
+                    xtype: 'checkbox',
+                    fieldLabel: 'Allow Unknown Contigs',
+                    helpPopup: 'By default, if the chain file contains contigs that cannot be mapped to contigs in the genomes, it will fail. If this is checked, these chains will simply be discarded.',
+                    name: 'allowUnknownContig',
+                    checked: false
                 },{
                     xtype: 'hidden', name: 'X-LABKEY-CSRF', value: LABKEY.CSRF
                 }]
@@ -103,12 +115,24 @@ Ext4.define('SequenceAnalysis.window.ImportChainFileWindow', {
         this.down('form').submit({
             scope: this,
             timeout: 999999999,
-            success: function(){
+            success: function(form, action){
                 Ext4.Msg.hide();
 
+                var msg = null;
+                if (action && action.response && action.response.responseText){
+                    if (Ext4.String.startsWith(action.response.responseText, '{')) {
+                        action.response.responseJSON = Ext4.decode(action.response.responseText);
+                    }
+
+                    if (action.response.responseJSON && action.response.responseJSON.messages) {
+                        console.log(action.response.responseJSON.messages);
+                        msg = action.response.responseJSON.messages.join('<br>');
+                    }
+                }
+
                 this.close();
 
-                Ext4.Msg.alert('Success', 'Chain File Imported!', function(){
+                Ext4.Msg.alert('Success', 'Chain File Imported!' + (msg ? ' Notes:<br><br>' + msg : ''), function(){
                     var dataRegion = LABKEY.DataRegions[this.dataRegionName];
                     dataRegion.refresh();
                 }, this);
@@ -118,7 +142,16 @@ Ext4.define('SequenceAnalysis.window.ImportChainFileWindow', {
                 var msg;
                 var serverMsg = [];
                 if (action && action.response && action.response.responseText){
-                    msg = action.response.responseText;
+                    if (Ext4.String.startsWith(action.response.responseText, '{')) {
+                        action.response.responseJSON = Ext4.decode(action.response.responseText);
+                    }
+
+                    if (action.response.responseJSON && action.response.responseJSON.exception) {
+                        msg = action.response.responseJSON.exception;
+                    }
+                    else {
+                        msg = action.response.responseText;
+                    }
                     serverMsg.push(action.response.responseText);
                 }