Merge pull request #95 from LabKey/fb_merge_discvr-20.11

Merge discvr-20.11 to develop

Author: bbimber

.github/workflows/build.yml

@@ -20,3 +20,12 @@ jobs:
           # NOTE: permissions are limited on the default secrets.GITHUB_TOKEN, including updating workflows, so use a personal access token
           github_token: ${{ secrets.PAT }}
 
+      - name: Publish Release
+
+        uses: "marvinpinto/action-automatic-releases@latest"
+        with:
+          repo_token: "${{ secrets.PAT }}"
+          prerelease: true
+          title: "Release "
+          files: |
+            /home/runner/work/_temp/_github_home/lkDist/discvr/DISCVR-*

.github/workflows/build_latest.yml

@@ -40,5 +40,5 @@ jobs:
           repo_token: "${{ secrets.PAT }}"
           automatic_release_tag: "latest"
           prerelease: true
-          title: "DISCVR Development Build: ${{ fromJson(steps.get_default_branch.outputs.data).default_branch }}"
+          title: "Development Build: ${{ fromJson(steps.get_default_branch.outputs.data).default_branch }}"
           files: /home/runner/work/_temp/_github_home/lkDist/discvr/DISCVR-*

.github/workflows/release.yml

@@ -2,14 +2,13 @@ name: Publish Release
 on:
   release:
     types: [created]
-    tags-ignore:
-      - 'latest'
 jobs:
   generate:
     name: Create release-artifacts
     # See: https://help.github.com/en/actions/reference/contexts-and-expression-syntax-for-github-actions#github-context
     # https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables#default-environment-variables
-    if: github.repository == 'BimberLab/DiscvrLabKeyModules'
+    # Note: 'latest' is a tag created by branch_latest.yml, which is the latest development build. This should not trigger normal releases.
+    if: github.repository == 'BimberLab/DiscvrLabKeyModules' && github.ref != 'refs/tags/latest'
     runs-on: ubuntu-latest
     steps:
       - name: "Build DISCVR"

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequenceOutputHandler.java

@@ -75,6 +75,14 @@ default boolean requiresSingleGenome()
         return true;
     }
 
+    /**
+     * @return Whether this handler requires each input to be associated with a genome
+     */
+    default boolean requiresGenome()
+    {
+        return true;
+    }
+
     public boolean canProcess(SequenceOutputFile o);
 
     /**

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/VariantProcessingStep.java

@@ -38,6 +38,14 @@ default void init(PipelineJob job, SequenceAnalysisJobSupport support, List<Sequ
 
     }
 
+    enum ScatterGatherMethod
+    {
+        none(),
+        contig(),
+        chunked(),
+        fixedJobs()
+    }
+
     public static interface Output extends PipelineStepOutput
     {
         public File getVCF();
@@ -50,7 +58,10 @@ public static interface RequiresPedigree
 
     public static interface SupportsScatterGather
     {
+        default void validateScatter(ScatterGatherMethod method, PipelineJob job) throws IllegalArgumentException
+        {
 
+        }
     }
 
     public static interface MayRequirePrepareTask

SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceAnalysisPanel.js

@@ -255,8 +255,8 @@ Ext4.define('SequenceAnalysis.panel.SequenceAnalysisPanel', {
             version: 2
         };
 
-        //first add the general params
-        Ext4.apply(json, this.down('#runInformation').getForm().getFieldValues());
+        //first add the general params. Note: include all to ensure we include unchecked checkboxes. Using useDataValues=false to ensure we get the string-serialized value
+        Ext4.apply(json, this.down('#runInformation').getForm().getValues(false, false, false, false));
 
         json['alignment.doAlignment'] = this.down('#doAlignment').getValue();
 
@@ -305,13 +305,15 @@ Ext4.define('SequenceAnalysis.panel.SequenceAnalysisPanel', {
                 helpPopup: 'Check to delete the intermediate files created by this pipeline.  In general these are not needed and it will save disk space.  These files might be useful for debugging though.',
                 name: 'deleteIntermediateFiles',
                 inputValue: true,
+                uncheckedValue: false,
                 checked: true,
                 xtype: 'checkbox'
             },{
                 fieldLabel: 'Copy Inputs To Working Directory?',
                 helpPopup: 'Check to copy the input files to the working directory.  Depending on your environment, this may or may not help performance.',
                 name: 'copyInputsLocally',
                 inputValue: true,
+                uncheckedValue: false,
                 checked: false,
                 xtype: 'checkbox'
             }, this.getSaveTemplateCfg(),{
@@ -320,6 +322,7 @@ Ext4.define('SequenceAnalysis.panel.SequenceAnalysisPanel', {
                 hidden: !LABKEY.Security.currentUser.isAdmin,
                 name: 'submitJobToReadsetContainer',
                 inputValue: true,
+                uncheckedValue: false,
                 checked: false,
                 xtype: 'checkbox'
             }]

SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js

@@ -407,12 +407,11 @@ Ext4.define('SequenceAnalysis.panel.VariantProcessingPanel', {
 			return;
 		}
 
-		Ext4.Msg.wait('Submitting...');
 		var json = {
 			handlerClass: 'org.labkey.sequenceanalysis.' + (this.showGenotypeGVCFs ? 'analysis.GenotypeGVCFHandler' : 'pipeline.ProcessVariantsHandler'),
 			outputFileIds: this.outputFileIds,
 			params: Ext4.encode(values)
-		}
+		};
 
 		if (Ext4.isDefined(values.doSplitJobs)) {
 			json.doSplitJobs = !!values.doSplitJobs;
@@ -441,6 +440,7 @@ Ext4.define('SequenceAnalysis.panel.VariantProcessingPanel', {
 			}
 		}
 
+		Ext4.Msg.wait('Submitting...');
 		LABKEY.Ajax.request({
 			url: LABKEY.ActionURL.buildURL('sequenceanalysis', actionName),
 			jsonData: json,
@@ -465,8 +465,8 @@ Ext4.define('SequenceAnalysis.panel.VariantProcessingPanel', {
 			version: 2
 		};
 
-		//first add the general params
-		Ext4.apply(json, this.down('#runInformation').getForm().getFieldValues());
+		//first add the general params. Note: include all to ensure we include unchecked checkboxes. Using useDataValues=false to ensure we get the string-serialized value
+		Ext4.apply(json, this.down('#runInformation').getForm().getValues(false, false, false, false));
 
 		//then append each section
 		var sections = this.query('sequenceanalysis-analysissectionpanel');

SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantScatterGatherPanel.js

@@ -61,6 +61,7 @@ Ext4.define('SequenceAnalysis.panel.VariantScatterGatherPanel', {
                                 name: 'scatterGather.allowSplitChromosomes',
                                 fieldLabel: 'Allow Split Contigs',
                                 value: true,
+                                uncheckedValue: false,
                                 inputValue: true,
                                 helpPopup: 'If true, a given chromosome/contig can be split between jobs.  Otherwise chromosomes are always intact across jobs.'
                             });

SequenceAnalysis/resources/web/SequenceAnalysis/window/OutputHandlerWindow.js

@@ -250,7 +250,7 @@ Ext4.define('SequenceAnalysis.window.OutputHandlerWindow', {
             return;
         }
 
-        //Note: include all to ensure we include unchecked checkboxes
+        //Note: include all to ensure we include unchecked checkboxes. Using useDataValues=false to ensure we get the string-serialized value
         var params = this.down('form').getForm().getValues(false, false, false, false);
         var json = {
             handlerType: this.handlerType,

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java

@@ -114,6 +114,7 @@
 import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler;
 import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
 import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor;
+import org.labkey.api.sequenceanalysis.pipeline.VariantProcessingStep;
 import org.labkey.api.util.ExceptionUtil;
 import org.labkey.api.util.FileType;
 import org.labkey.api.util.FileUtil;
@@ -162,7 +163,6 @@
 import org.labkey.sequenceanalysis.run.util.FastqcRunner;
 import org.labkey.sequenceanalysis.util.ChainFileValidator;
 import org.labkey.sequenceanalysis.util.FastqUtils;
-import org.labkey.sequenceanalysis.util.ScatterGatherUtils;
 import org.labkey.sequenceanalysis.util.SequenceUtil;
 import org.labkey.sequenceanalysis.visualization.VariationChart;
 import org.springframework.beans.PropertyValues;
@@ -4053,13 +4053,37 @@ public ApiResponse execute(RunSequenceHandlerForm form, BindException errors) th
                 errors.reject(ERROR_MSG, "Unable to parse JSON params: " + e.getMessage());
                 return null;
             }
+            catch (IllegalArgumentException e)
+            {
+                errors.reject(ERROR_MSG, e.getMessage());
+                return null;
+            }
         }
 
         protected PipelineJob createOutputJob(RunSequenceHandlerForm form, Container targetContainer, String jobName, PipeRoot pr1, SequenceOutputHandler handler, List<SequenceOutputFile> inputs, JSONObject json) throws IOException, PipelineJobException
         {
+            validateGenomes(inputs, handler);
             return new SequenceOutputHandlerJob(targetContainer, getUser(), jobName, pr1, handler, inputs, json);
         }
 
+        protected void validateGenomes(List<SequenceOutputFile> inputs, SequenceOutputHandler<?> handler) throws IllegalArgumentException
+        {
+            Set<Integer> genomes = new HashSet<>();
+            inputs.forEach(x -> {
+                if (x.getLibrary_id() == null && (handler.requiresGenome() || handler.requiresSingleGenome()))
+                {
+                    throw new IllegalArgumentException("Input missing genome: " + x.getRowid());
+                }
+
+                genomes.add(x.getLibrary_id());
+            });
+
+            if (genomes.size() > 1 && handler.requiresSingleGenome())
+            {
+                throw new IllegalArgumentException("All inputs must use the same base genome");
+            }
+        }
+
         private PipeRoot getPipeRoot(Container targetContainer, Map<Container, PipeRoot> containerToPipeRootMap)
         {
             if (!containerToPipeRootMap.containsKey(targetContainer))
@@ -4078,12 +4102,14 @@ private PipeRoot getPipeRoot(Container targetContainer, Map<Container, PipeRoot>
     @RequiresPermission(InsertPermission.class)
     public class RunVariantProcessingAction extends RunSequenceHandlerAction
     {
+        @Override
         protected PipelineJob createOutputJob(RunSequenceHandlerForm form, Container targetContainer, String jobName, PipeRoot pr1, SequenceOutputHandler handler, List<SequenceOutputFile> inputs, JSONObject json) throws PipelineJobException, IOException
         {
             String method = json.getString("scatterGatherMethod");
             try
             {
-                ScatterGatherUtils.ScatterGatherMethod scatterMethod = ScatterGatherUtils.ScatterGatherMethod.valueOf(method);
+                validateGenomes(inputs, handler);
+                VariantProcessingStep.ScatterGatherMethod scatterMethod = VariantProcessingStep.ScatterGatherMethod.valueOf(method);
 
                 return new VariantProcessingJob(targetContainer, getUser(), jobName, pr1, handler, inputs, json, scatterMethod);
             }