Merge pull request #100 from LabKey/fb_merge_discvr-21.3

Merge discvr-21.3 to develop

Author: bbimber

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/DefaultPipelineStepOutput.java

@@ -105,9 +105,24 @@ public List<SequenceOutput> getSequenceOutputs()
     @Override
     public void addSequenceOutput(File file, String label, String category, @Nullable Integer readsetId, @Nullable Integer analysisId, @Nullable Integer genomeId, @Nullable String description)
     {
+        _intermediateFiles.remove(file);
+
         _sequenceOutputs.add(new SequenceOutput(file, label, category, readsetId, analysisId, genomeId, description));
     }
 
+    private boolean existsAsOutput(File f)
+    {
+        for (SequenceOutput so  : _sequenceOutputs)
+        {
+            if (so.getFile() != null && so.getFile().equals(f))
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     @Override
     public void addInput(File input, String role)
     {
@@ -132,13 +147,16 @@ public void addIntermediateFile(File file, String role)
         if (role != null)
             addOutput(file, role);
 
-        _intermediateFiles.add(file);
+        if (!existsAsOutput(file))
+        {
+            _intermediateFiles.add(file);
+        }
     }
 
     @Override
     public void addIntermediateFiles(Collection<File> files)
     {
-        _intermediateFiles.addAll(files);
+        files.forEach(this::addIntermediateFile);
     }
 
     public void addPicardMetricsFile(Readset rs, File metricFile, File inputFile)

SequenceAnalysis/resources/views/sequenceAnalysisDetails.html

@@ -68,7 +68,7 @@
         queryName: 'quality_metrics',
         maxRows: 20,
         containerPath: Laboratory.Utils.getQueryContainerPath(),
-        columns: 'fileid,metricname,metricvalue,qualvalue,comment,workbook,readset,runid',
+        columns: 'dataid,metricname,metricvalue,qualvalue,comment,workbook,readset,runid',
         filterArray: [LABKEY.Filter.create('analysis_id', analysisId, LABKEY.Filter.Types.EQUAL)]
     }).render('qualityMetrics_'+webpart.wrapperDivId);
 

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerFinalTask.java

@@ -87,7 +87,8 @@ public RecordedActionSet run() throws PipelineJobException
         //create analysisRecord
         AnalysisModelImpl am = new AnalysisModelImpl();
         am.setContainer(getJob().getContainerId());
-        am.setDescription(getPipelineJob().getDescription());
+        String description = getPipelineJob().getParameters().getOrDefault("jobDescription", null) != null ? getPipelineJob().getParameters().get("jobDescription") : getPipelineJob().getDescription();
+        am.setDescription(description);
         am.setRunId(runId);
 
 

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/TaskFileManagerImpl.java

@@ -401,7 +401,27 @@ private File convertRelPathToFile(String line)
     public void addIntermediateFile(File f)
     {
         _job.getLogger().debug("adding intermediate file: " + f.getPath());
-        _intermediateFiles.add(f);
+        if (existsAsOutput(f))
+        {
+            _job.getLogger().debug("file exists as sequence output, will not put into intermediate files");
+        }
+        else
+        {
+            _intermediateFiles.add(f);
+        }
+    }
+
+    private boolean existsAsOutput(File f)
+    {
+        for (SequenceOutputFile so : _outputsToCreate)
+        {
+            if (so.getFile() != null && so.getFile().equals(f))
+            {
+                return true;
+            }
+        }
+
+        return false;
     }
 
     @Override

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/PlinkPcaStep.java

@@ -1,6 +1,9 @@
 package org.labkey.sequenceanalysis.run.variant;
 
+import au.com.bytecode.opencsv.CSVWriter;
+import htsjdk.samtools.util.IOUtil;
 import htsjdk.samtools.util.Interval;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.json.JSONObject;
 import org.labkey.api.pipeline.PipelineJobException;
@@ -18,6 +21,7 @@
 
 import javax.annotation.Nullable;
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -36,8 +40,10 @@ public Provider()
             super("PlinkPcaStep", "Plink/PCA", "", "This will run plink to generate the data for MDS/PCA", Arrays.asList(
                     ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("--not-chr"), "excludedContigs", "Excluded Contigs", "A comma separated list of contigs to exclude, such as X,Y,MT.", "textfield", new JSONObject(){{
 
-                    }}, "X,Y,MT")
-            ), null, "https://zzz.bwh.harvard.edu/plink/");
+                    }}, "X,Y,MT"),
+                    ToolParameterDescriptor.create(SelectSamplesStep.SAMPLE_INCLUDE, "Sample(s) Include", "Only the following samples will be included in the analysis.", "sequenceanalysis-trimmingtextarea", null, null),
+                    ToolParameterDescriptor.create(SelectSamplesStep.SAMPLE_EXCLUDE, "Samples(s) To Exclude", "The following samples will be excluded from the analysis.", "sequenceanalysis-trimmingtextarea", null, null)
+            ), Arrays.asList("sequenceanalysis/field/TrimmingTextArea.js"), "https://zzz.bwh.harvard.edu/plink/");
         }
 
         public PlinkPcaStep create(PipelineContext ctx)
@@ -46,6 +52,30 @@ public PlinkPcaStep create(PipelineContext ctx)
         }
     }
 
+    private void addSubjectSelectOptions(String text, List<String> args, String argName, File outputFile, VariantProcessingStepOutputImpl output) throws PipelineJobException
+    {
+        text = StringUtils.trimToNull(text);
+        if (text != null)
+        {
+            String[] names = text.split(";");
+            try (CSVWriter writer = new CSVWriter(IOUtil.openFileForBufferedUtf8Writing(outputFile), '\t', CSVWriter.NO_QUOTE_CHARACTER))
+            {
+                Arrays.stream(names).forEach(x -> {
+                    writer.writeNext(new String[]{x, x});
+                });
+            }
+            catch (IOException e)
+            {
+                throw new PipelineJobException(e);
+            }
+
+            args.add(argName);
+            args.add(outputFile.getPath());
+
+            output.addIntermediateFile(outputFile);
+        }
+    }
+
     @Override
     public Output processVariants(File inputVCF, File outputDirectory, ReferenceGenome genome, @Nullable List<Interval> intervals) throws PipelineJobException
     {
@@ -55,7 +85,13 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
         args.add(getWrapper().getExe().getPath());
         args.add("--pca");
         args.add("--allow-extra-chr");
-        args.add("--keep WGS.names");
+
+        String samplesToInclude = getProvider().getParameterByName(SelectSamplesStep.SAMPLE_INCLUDE).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class);
+        addSubjectSelectOptions(samplesToInclude, args, "--keep", new File(outputDirectory, "samplesToKeep.txt"), output);
+
+        String samplesToExclude = getProvider().getParameterByName(SelectSamplesStep.SAMPLE_EXCLUDE).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class);
+        addSubjectSelectOptions(samplesToExclude, args, "--exclude", new File(outputDirectory, "samplesToExclude.txt"), output);
+
         args.add("--vcf");
         args.add(inputVCF.getPath());
 

singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java

@@ -100,6 +100,7 @@ public Output execute(SequenceOutputHandler.JobContext ctx, List<SeuratObjectWra
         File intermediates = new File(ctx.getOutputDir(), "intermediateFiles.txt");
         if (intermediates.exists())
         {
+            getPipelineCtx().getLogger().debug("inspecting intermediateFiles.txt");
             try (CSVReader reader = new CSVReader(Readers.getReader(intermediates), '\t'))
             {
                 String[] line;
@@ -324,6 +325,11 @@ else if ("sequenceanalysis-trimmingtextarea".equals(pd.getFieldXtype()))
             String[] vals = val.split(",");
             return "c('" + StringUtils.join(vals, "','") + "')";
         }
+        else if (pd.isMultiValue())
+        {
+            String[] vals = val.split(pd.getDelimiter());
+            return "c('" + StringUtils.join(vals, "','") + "')";
+        }
 
         return  "'" + val + "'";
     }

singlecell/api-src/org/labkey/api/singlecell/pipeline/SeuratToolParameter.java

@@ -11,23 +11,31 @@ public class SeuratToolParameter extends ToolParameterDescriptor
 {
     private String _rName;
     private boolean _includeIfEmptyOrNull;
+    private String _delimiter = ";";
+    private boolean _isMultiValue;
 
-    public SeuratToolParameter(String name, String label, String description, String fieldXtype, @Nullable Object defaultValue, @Nullable JSONObject additionalExtConfig, String rName, boolean includeIfEmptyOrNull)
+    public SeuratToolParameter(String name, String label, String description, String fieldXtype, @Nullable Object defaultValue, @Nullable JSONObject additionalExtConfig, String rName, boolean includeIfEmptyOrNull, boolean isMultiValue)
     {
         super(null, name, label, description, fieldXtype, defaultValue, additionalExtConfig);
 
         _rName = rName;
         _includeIfEmptyOrNull = includeIfEmptyOrNull;
+        _isMultiValue = isMultiValue;
     }
 
     public static SeuratToolParameter create(String name, String label, String description, String fieldXtype, @Nullable JSONObject additionalExtConfig, @Nullable Object defaultValue)
     {
-        return new SeuratToolParameter(name, label, description, fieldXtype, defaultValue, additionalExtConfig, null, true);
+        return new SeuratToolParameter(name, label, description, fieldXtype, defaultValue, additionalExtConfig, null, true, false);
     }
 
     public static SeuratToolParameter create(String name, String label, String description, String fieldXtype, @Nullable JSONObject additionalExtConfig, @Nullable Object defaultValue, String rName, boolean includeIfEmptyOrNull)
     {
-        return new SeuratToolParameter(name, label, description, fieldXtype, defaultValue, additionalExtConfig, rName, includeIfEmptyOrNull);
+        return SeuratToolParameter.create(name, label, description, fieldXtype, additionalExtConfig, defaultValue, rName, includeIfEmptyOrNull, false);
+    }
+
+    public static SeuratToolParameter create(String name, String label, String description, String fieldXtype, @Nullable JSONObject additionalExtConfig, @Nullable Object defaultValue, String rName, boolean includeIfEmptyOrNull, boolean isMultiValue)
+    {
+        return new SeuratToolParameter(name, label, description, fieldXtype, defaultValue, additionalExtConfig, rName, includeIfEmptyOrNull, isMultiValue);
     }
 
     public String getVariableName()
@@ -44,4 +52,14 @@ public boolean shouldIncludeInMarkdown(PipelineJob job, PipelineStepProvider pro
 
         return true;
     }
+
+    public boolean isMultiValue()
+    {
+        return _isMultiValue;
+    }
+
+    public String getDelimiter()
+    {
+        return _delimiter;
+    }
 }

singlecell/src/org/labkey/singlecell/CellHashingServiceImpl.java

@@ -905,6 +905,14 @@ public File generateCellHashingCalls(File citeSeqCountOutDir, File outputDir, St
         }
 
         File htmlFile = new File(outputDir, basename + ".html");
+        File localHtml = new File(localPipelineDir, htmlFile.getName());
+
+        // Note: if this job fails and then is resumed, having that pre-existing copy of the HTML can pose a problem
+        if (localHtml.exists())
+        {
+            log.debug("Deleting pre-existing HTML file: " + localHtml.getPath());
+        }
+
         File callsFile = new File(outputDir, basename + CALL_EXTENSION);
         File metricsFile = getMetricsFile(callsFile);
 
@@ -1011,7 +1019,6 @@ public File generateCellHashingCalls(File citeSeqCountOutDir, File outputDir, St
             {
                 try
                 {
-                    File localHtml = new File(localPipelineDir, htmlFile.getName());
                     log.info("copying HTML file locally for easier debugging: " + localHtml.getPath());
                     if (localHtml.exists())
                     {

singlecell/src/org/labkey/singlecell/pipeline/singlecell/FindMarkers.java

@@ -37,7 +37,7 @@ public Provider()
                         put("initialValues", "wilcox;MAST;DESeq2");
                         put("delimiter", ";");
                         put("joinReturnValue", true);
-                    }}, null),
+                    }}, null, null, false, true),
                     SeuratToolParameter.create("pValThreshold", "pVal Threshold", "Only genes with adjusted p-values below this will be reported", "ldk-numberfield", new JSONObject(){{
                         put("minValue", 0);
                         put("decimalPrecision", 5);