Merge pull request #223 from BimberLab/23.7_fb_merge

Merge discvr-23.3 to discvr-23.7

Author: bbimber

.gitignore

@@ -3,6 +3,7 @@
 /jbrowse/node_modules
 /intellijBuild
 /travisSettings.sh
+index/
 
 # Created by npm install
 jbrowse/jb_run.js

SequenceAnalysis/resources/queries/sequenceanalysis/alignment_summary_by_lineage.query.xml

@@ -38,6 +38,10 @@
                             </conditionalFormat>
                         </conditionalFormats>
                     </column>
+                    <column columnName="rowids">
+                        <columnTitle>RowIds</columnTitle>
+                        <isHidden>true</isHidden>
+                    </column>
                 </columns>
             </table>
         </tables>

SequenceAnalysis/resources/queries/sequenceanalysis/alignment_summary_by_lineage.sql

@@ -34,6 +34,7 @@ select
       SELECT distinct asj.alignment_id from sequenceanalysis.alignment_summary_junction asj WHERE asj.ref_nt_id.locus = a.loci and asj.status = true
     )
   ) as float)), 2) as percent_from_locus,
+  group_concat(distinct a.rowid, ',') as rowids
 
 FROM (
 

SequenceAnalysis/resources/queries/sequenceanalysis/alignment_summary_grouped.query.xml

@@ -50,6 +50,18 @@
                             </conditionalFormat>
                         </conditionalFormats>
                     </column>
+                    <column columnName="rowids">
+                        <columnTitle>RowIds</columnTitle>
+                        <isHidden>true</isHidden>
+                    </column>
+                    <column columnName="nAlignments">
+                        <columnTitle># Alignments</columnTitle>
+                        <isHidden>true</isHidden>
+                    </column>
+                    <column columnName="nLoci">
+                        <columnTitle># Loci</columnTitle>
+                        <isHidden>true</isHidden>
+                    </column>
                 </columns>
             </table>
         </tables>

SequenceAnalysis/resources/queries/sequenceanalysis/alignment_summary_grouped.sql

@@ -33,7 +33,7 @@ select
 --     else 'Minor'
 --   end as category,
 
-  group_concat(a.rowid) as rowids,
+  group_concat(a.rowid, ',') as rowids,
   group_concat(distinct a.haplotypesWithAllele) as haplotypesWithAllele,
 
   CAST((select sum(s.total) as total FROM sequenceanalysis.alignment_summary s WHERE s.analysis_id = a.analysis_id AND s.rowid IN (
@@ -45,7 +45,9 @@ select
       SELECT distinct asj.alignment_id from sequenceanalysis.alignment_summary_junction asj WHERE asj.ref_nt_id.locus = a.loci and asj.status = true
     )
   ) as float) END), 2) as percent_from_locus,
-  max(lastModified) as lastModified
+  max(lastModified) as lastModified,
+  count(distinct a.rowid) as nAlignments,
+  max(a.nloci) as nLoci
 
 FROM (
 
@@ -58,6 +60,7 @@ FROM (
     group_concat(distinct j.ref_nt_id.lineage, chr(10)) as lineages,
     count(distinct j.ref_nt_id.lineage) as totalLineages,
     group_concat(distinct j.ref_nt_id.locus, chr(10)) as loci,
+    count(distinct j.ref_nt_id.locus) as nloci,
     group_concat(distinct hs.haplotype, chr(10)) as haplotypesWithAllele,
 
     total,

SequenceAnalysis/resources/queries/sequenceanalysis/sequence_readsets/.qview.xml

@@ -11,6 +11,7 @@
        <column name="comments" />
        <column name="barcode5" />
        <column name="barcode3" />
+       <column name="concentration" />
        <column name="sampleid" />
 
        <column name="instrument_run_id" />

SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantFilterPanel.js

@@ -65,6 +65,18 @@ Ext4.define('SequenceAnalysis.panel.VariantFilterPanel', {
 						handler: function(gridBtn){
 							this.addFilter('Depth100', 'vc.hasAttribute(\'DP\') && DP > 100.0');
 						}
+					},{
+						text: 'High IMPACT',
+						scope: this,
+						handler: function(gridBtn){
+							this.addFilter('High-IMPACT', 'IMPACT == \'HIGH\'');
+						}
+					},{
+						text: 'CADD PH',
+						scope: this,
+						handler: function(gridBtn){
+							this.addFilter('CADD_PH', 'CADD_PH > 25');
+						}
 					},{
 						text: 'No Genotypes Called',
 						scope: this,
@@ -100,7 +112,7 @@ Ext4.define('SequenceAnalysis.panel.VariantFilterPanel', {
 					editor: {
 						xtype: 'textfield',
 						allowBlank: false,
-						stripCharsRe: /(^['"]+)|(['"]+$)/g
+						stripCharsRe: /(^["]+)|(["]+$)/g
 					}
 				//}, {
 				//	dataIndex: 'invert',

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java

@@ -501,7 +501,7 @@ private static String countUsingBcfTools(File vcf, Logger log) throws PipelineJo
     {
         BcftoolsRunner wrapper = new BcftoolsRunner(log);
 
-        return wrapper.executeWithOutput(Arrays.asList(wrapper.getBcfToolsPath().getPath(), "index", "-n", vcf.getPath()));
+        return StringUtils.trimToNull(wrapper.executeWithOutput(Arrays.asList(wrapper.getBcfToolsPath().getPath(), "index", "-n", vcf.getPath())));
     }
 
     public static String getVCFLineCount(File vcf, Logger log, boolean passOnly, boolean useBcfTools) throws PipelineJobException
@@ -528,7 +528,7 @@ public static String getVCFLineCount(File vcf, Logger log, boolean passOnly, boo
         String cat = vcf.getName().endsWith(".gz") ? "zcat" : "cat";
         SimpleScriptWrapper wrapper = new SimpleScriptWrapper(null);
 
-        String ret = wrapper.executeWithOutput(Arrays.asList("/bin/bash", "-c", cat + " \"" + vcf.getPath() + "\" | grep -v \"#\" | " + (passOnly ? "awk ' $7 == \"PASS\" || $7 == \"\\.\" ' | " : "") + "wc -l | awk \" { print $1 } \""));
+        String ret = StringUtils.trimToNull(wrapper.executeWithOutput(Arrays.asList("/bin/bash", "-c", cat + " \"" + vcf.getPath() + "\" | grep -v \"#\" | " + (passOnly ? "awk ' $7 == \"PASS\" || $7 == \"\\.\" ' | " : "") + "wc -l | awk \" { print $1 } \"")));
 
         //NOTE: unsure how to get awk to omit this warning, so discard it:
         //the warning is: escape '\.' treated as plain '.'
@@ -877,10 +877,9 @@ public void performAdditionalMergeTasks(JobContext ctx, PipelineJob job, TaskFil
         List<PipelineStepCtx<VariantProcessingStep>> providers = SequencePipelineService.get().getSteps(job, VariantProcessingStep.class);
         for (PipelineStepCtx<VariantProcessingStep> stepCtx : providers)
         {
-            VariantProcessingStep step = stepCtx.getProvider().create(ctx);
-            if (step instanceof VariantProcessingStep.SupportsScatterGather)
+            if (stepCtx.getProvider() instanceof VariantProcessingStep.SupportsScatterGather ssg)
             {
-                ((VariantProcessingStep.SupportsScatterGather)step).performAdditionalMergeTasks(ctx, job, manager, genome, orderedScatterOutputs);
+                ssg.performAdditionalMergeTasks(ctx, job, manager, genome, orderedScatterOutputs);
             }
         }
     }

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerJob.java

@@ -160,7 +160,7 @@ public File getSerializedOutputFilesFile()
     }
 
     @Override
-    public TaskPipeline getTaskPipeline()
+    public TaskPipeline<?> getTaskPipeline()
     {
         return  PipelineJobService.get().getTaskPipeline(new TaskId(SequenceOutputHandlerJob.class));
     }

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingRemoteMergeTask.java

@@ -237,6 +237,7 @@ private File runDefaultVariantMerge(JobContextImpl ctx, TaskFileManagerImpl mana
 
         if (handler instanceof VariantProcessingStep.SupportsScatterGather)
         {
+            ctx.getLogger().debug("Running additional merge tasks");
             ((VariantProcessingStep.SupportsScatterGather) handler).performAdditionalMergeTasks(ctx, getPipelineJob(), manager, genome, toConcat);
         }