Merge pull request #33 from BimberLab/fb_merge_discvr-20.11

Merge discvr-20.11 to discvr-21.3

Author: bbimber

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/DefaultPipelineStepOutput.java

@@ -105,9 +105,24 @@ public List<SequenceOutput> getSequenceOutputs()
     @Override
     public void addSequenceOutput(File file, String label, String category, @Nullable Integer readsetId, @Nullable Integer analysisId, @Nullable Integer genomeId, @Nullable String description)
     {
+        _intermediateFiles.remove(file);
+
         _sequenceOutputs.add(new SequenceOutput(file, label, category, readsetId, analysisId, genomeId, description));
     }
 
+    private boolean existsAsOutput(File f)
+    {
+        for (Pair<File, String> pair : _outputs)
+        {
+            if (pair.first != null && pair.first.equals(f))
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     @Override
     public void addInput(File input, String role)
     {
@@ -132,13 +147,16 @@ public void addIntermediateFile(File file, String role)
         if (role != null)
             addOutput(file, role);
 
-        _intermediateFiles.add(file);
+        if (!existsAsOutput(file))
+        {
+            _intermediateFiles.add(file);
+        }
     }
 
     @Override
     public void addIntermediateFiles(Collection<File> files)
     {
-        _intermediateFiles.addAll(files);
+        files.forEach(this::addIntermediateFile);
     }
 
     public void addPicardMetricsFile(Readset rs, File metricFile, File inputFile)

SequenceAnalysis/resources/views/sequenceAnalysisDetails.html

@@ -68,7 +68,7 @@
         queryName: 'quality_metrics',
         maxRows: 20,
         containerPath: Laboratory.Utils.getQueryContainerPath(),
-        columns: 'fileid,metricname,metricvalue,qualvalue,comment,workbook,readset,runid',
+        columns: 'dataid,metricname,metricvalue,qualvalue,comment,workbook,readset,runid',
         filterArray: [LABKEY.Filter.create('analysis_id', analysisId, LABKEY.Filter.Types.EQUAL)]
     }).render('qualityMetrics_'+webpart.wrapperDivId);
 

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/TaskFileManagerImpl.java

@@ -401,7 +401,27 @@ private File convertRelPathToFile(String line)
     public void addIntermediateFile(File f)
     {
         _job.getLogger().debug("adding intermediate file: " + f.getPath());
-        _intermediateFiles.add(f);
+        if (existsAsOutput(f))
+        {
+            _job.getLogger().debug("file exists as sequence output, will not put into intermediate files");
+        }
+        else
+        {
+            _intermediateFiles.add(f);
+        }
+    }
+
+    private boolean existsAsOutput(File f)
+    {
+        for (SequenceOutputFile so : _outputsToCreate)
+        {
+            if (so.getFile() != null && so.getFile().equals(f))
+            {
+                return true;
+            }
+        }
+
+        return false;
     }
 
     @Override

singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java

@@ -100,6 +100,7 @@ public Output execute(SequenceOutputHandler.JobContext ctx, List<SeuratObjectWra
         File intermediates = new File(ctx.getOutputDir(), "intermediateFiles.txt");
         if (intermediates.exists())
         {
+            getPipelineCtx().getLogger().debug("inspecting intermediateFiles.txt");
             try (CSVReader reader = new CSVReader(Readers.getReader(intermediates), '\t'))
             {
                 String[] line;

singlecell/src/org/labkey/singlecell/CellHashingServiceImpl.java

@@ -905,6 +905,14 @@ public File generateCellHashingCalls(File citeSeqCountOutDir, File outputDir, St
         }
 
         File htmlFile = new File(outputDir, basename + ".html");
+        File localHtml = new File(localPipelineDir, htmlFile.getName());
+
+        // Note: if this job fails and then is resumed, having that pre-existing copy of the HTML can pose a problem
+        if (localHtml.exists())
+        {
+            log.debug("Deleting pre-existing HTML file: " + localHtml.getPath());
+        }
+
         File callsFile = new File(outputDir, basename + CALL_EXTENSION);
         File metricsFile = getMetricsFile(callsFile);
 
@@ -1011,7 +1019,6 @@ public File generateCellHashingCalls(File citeSeqCountOutDir, File outputDir, St
             {
                 try
                 {
-                    File localHtml = new File(localPipelineDir, htmlFile.getName());
                     log.info("copying HTML file locally for easier debugging: " + localHtml.getPath());
                     if (localHtml.exists())
                     {