Merge pull request #97 from LabKey/fb_merge_discvr-20.11

bbimber · web-flow · commit ab7cbd9290da · 2021-03-03T06:02:18.000-08:00
Merge discvr-20.11 to develop
diff --git a/SequenceAnalysis/resources/queries/sequenceanalysis/reference_library_members/.qview.xml b/SequenceAnalysis/resources/queries/sequenceanalysis/reference_library_members/.qview.xml
@@ -2,6 +2,7 @@
    <columns>
        <column name="library_id" />
        <column name="ref_nt_id" />
+       <column name="alias" />
        <column name="start" />
        <column name="stop" />
        <column name="type" />
diff --git a/SequenceAnalysis/resources/schemas/dbscripts/postgresql/SequenceAnalysis-12.324-12.325.sql b/SequenceAnalysis/resources/schemas/dbscripts/postgresql/SequenceAnalysis-12.324-12.325.sql
@@ -0,0 +1 @@
+ALTER TABLE sequenceanalysis.reference_library_members ADD alias VARCHAR(4000);
diff --git a/SequenceAnalysis/resources/schemas/dbscripts/sqlserver/SequenceAnalysis-12.324-12.325.sql b/SequenceAnalysis/resources/schemas/dbscripts/sqlserver/SequenceAnalysis-12.324-12.325.sql
@@ -0,0 +1 @@
+ALTER TABLE sequenceanalysis.reference_library_members ADD alias VARCHAR(4000);
diff --git a/SequenceAnalysis/resources/schemas/sequenceanalysis.xml b/SequenceAnalysis/resources/schemas/sequenceanalysis.xml
@@ -2234,6 +2234,9 @@
                     <fkColumnName>rowid</fkColumnName>
                 </fk>
             </column>
+            <column columnName="alias">
+                <description>Optional.  Can be used to give the sequence a local name</description>
+            </column>
             <column columnName="start">
                 <description>Optional.  Can be used in conjunction with stop to select a subsequence of the reference.  If blank the region will begin at the first base of the reference</description>
             </column>
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java
@@ -178,7 +178,7 @@ public String getName()
     @Override
     public Double getSchemaVersion()
     {
-        return 12.324;
+        return 12.325;
     }
 
     @Override
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportGenomeTrackTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportGenomeTrackTask.java
@@ -357,6 +357,8 @@ private Map<String, Pair<String, Integer>> getNameTranslationMap(List<RefNtSeque
 
         getJob().getLogger().info("building list of chromosome name translations");
         Map<String, Pair<String, Integer>> ret = new CaseInsensitiveHashMap<>();
+
+        TableInfo rlm = SequenceAnalysisSchema.getTable(SequenceAnalysisSchema.TABLE_REF_LIBRARY_MEMBERS);
         for (RefNtSequenceModel m : refNtSequenceModels)
         {
             if (m.getOffsetsFile() != null && m.getOffsetsFile().exists())
@@ -446,6 +448,18 @@ private Map<String, Pair<String, Integer>> getNameTranslationMap(List<RefNtSeque
                         String strPart = m.getName().toLowerCase().replaceAll("^chr", "");
                         ret.put(strPart, Pair.of(m.getName(), 0));
                     }
+
+                    SimpleFilter filter = new SimpleFilter(FieldKey.fromString("library_id"), getPipelineJob().getLibraryId());
+                    filter.addCondition(FieldKey.fromString("ref_nt_id"), m.getRowid());
+                    TableSelector ts = new TableSelector(rlm, PageFlowUtil.set("alias"), filter, null);
+                    if (ts.exists())
+                    {
+                        String alias = ts.getObject(String.class);
+                        if (alias != null)
+                        {
+                            ret.put(alias, Pair.of(m.getName(), 0));
+                        }
+                    }
                 }
             }
         }
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/AbstractGenomicsDBImportHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/AbstractGenomicsDBImportHandler.java
@@ -40,12 +40,14 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 abstract public class AbstractGenomicsDBImportHandler extends AbstractParameterizedOutputHandler<SequenceOutputHandler.SequenceOutputProcessor> implements SequenceOutputHandler.TracksVCF, SequenceOutputHandler.HasCustomVariantMerge, VariantProcessingStep.MayRequirePrepareTask, VariantProcessingStep.SupportsScatterGather
 {
@@ -302,16 +304,28 @@ Set<String> getContigsInInputs(List<File> inputVCFs, Logger log) throws Pipeline
         return _contigsInInputs;
     }
 
-    private void copyToLevelFiles(PipelineJob job, File sourceWorkspace, File destinationWorkspace, boolean overwrite) throws IOException
+    private void copyToLevelFiles(PipelineJob job, File sourceWorkspace, File destinationWorkspace, boolean removeOtherFiles, boolean overwriteExisting) throws IOException
     {
         job.getLogger().info("Copying top-level files from: " + sourceWorkspace.getPath());
+        if (removeOtherFiles)
+        {
+            for (File f : destinationWorkspace.listFiles())
+            {
+                if (!f.isDirectory())
+                {
+                    job.getLogger().debug("deleting existing top-level file: " + f.getPath());
+                    f.delete();
+                }
+            }
+        }
+
         for (String fn : Arrays.asList("callset.json", "vidmap.json", "vcfheader.vcf", "__tiledb_workspace.tdb"))
         {
             File source = new File(sourceWorkspace, fn);
             File dest = new File(destinationWorkspace, fn);
             if (dest.exists())
             {
-                if (!overwrite)
+                if (!overwriteExisting)
                 {
                     job.getLogger().debug("workspace file exists, will not overwrite: " + dest.getPath());
                     continue;
@@ -457,7 +471,7 @@ else if (genomeIds.isEmpty())
                 }
 
                 File sourceWorkspace = getSourceWorkspace(ctx.getParams(), ctx.getSequenceSupport());
-                copyWorkspace(ctx, sourceWorkspace, workingDestinationWorkspaceFolder, genome, toDelete, !genomicsDbCompleted);
+                copyWorkspace(ctx, sourceWorkspace, workingDestinationWorkspaceFolder, genome, toDelete, !genomicsDbCompleted, !genomicsDbCompleted, !genomicsDbCompleted);
             }
             else
             {
@@ -587,7 +601,7 @@ else if (genomeIds.isEmpty())
 
             if (!copyToSourceDone.exists())
             {
-                copyWorkspace(ctx, workingDestinationWorkspaceFolder, workspaceLocalDir, genome, toDelete, true);
+                copyWorkspace(ctx, workingDestinationWorkspaceFolder, workspaceLocalDir, genome, toDelete, true, false, false);
 
                 try
                 {
@@ -635,7 +649,7 @@ else if (genomeIds.isEmpty())
         }
     }
 
-    private void copyWorkspace(JobContext ctx, File sourceWorkspace, File destinationWorkspaceFolder, ReferenceGenome genome, Collection<File> toDelete, boolean alwaysPerformRsync) throws PipelineJobException
+    private void copyWorkspace(JobContext ctx, File sourceWorkspace, File destinationWorkspaceFolder, ReferenceGenome genome, Collection<File> toDelete, boolean alwaysPerformRsync, boolean overwriteTopLevelFiles, boolean removeExistingTopLevelFiles) throws PipelineJobException
     {
         if (!destinationWorkspaceFolder.exists())
         {
@@ -665,14 +679,15 @@ private void copyWorkspace(JobContext ctx, File sourceWorkspace, File destinatio
                     else
                     {
                         ctx.getLogger().info("has been copied, skipping: " + i.getContig());
+                        assertContigFoldersEqual(sourceFolder, destContigFolder);
                         reportFragmentsPerContig(ctx, destContigFolder, i.getContig());
                         continue;
                     }
                 }
 
                 if (!haveCopiedTopLevelFiles)
                 {
-                    copyToLevelFiles(ctx.getJob(), sourceWorkspace, destinationWorkspaceFolder, false);
+                    copyToLevelFiles(ctx.getJob(), sourceWorkspace, destinationWorkspaceFolder, removeExistingTopLevelFiles, overwriteTopLevelFiles);
                     haveCopiedTopLevelFiles = true;
                 }
 
@@ -704,6 +719,7 @@ private void copyWorkspace(JobContext ctx, File sourceWorkspace, File destinatio
                 }
 
                 FileUtils.touch(copyDone);
+                assertContigFoldersEqual(sourceFolder, destContigFolder);
                 reportFragmentsPerContig(ctx, destContigFolder, i.getContig());
             }
             catch (IOException e)
@@ -718,22 +734,46 @@ private File getCopyToSourceDone(JobContext ctx)
         return new File(ctx.getSourceDirectory(), "copyToWebserver.done");
     }
 
-    private void reportFragmentsPerContig(JobContext ctx, File destContigFolder, String contigName)
+    private void assertContigFoldersEqual(File sourceFolder, File destContigFolder) throws IllegalArgumentException
     {
-        if (destContigFolder.exists())
+        List<String> sourceFiles = getFragmentsPerContig(sourceFolder);
+        List<String> destFiles = getFragmentsPerContig(destContigFolder);
+
+        if (!sourceFiles.equals(destFiles))
         {
-            File[] children = destContigFolder.listFiles(x -> {
-                return  x.isDirectory() && !"genomicsdb_meta_dir".equals(x.getName());
-            });
+            throw new IllegalArgumentException("Source and destination contig files not equal for: " + destContigFolder.getPath());
+        }
+    }
 
-            ctx.getLogger().info(contigName + " total fragments: " + children.length);
+    private void reportFragmentsPerContig(JobContext ctx, File destContigFolder, String contigName)
+    {
+        List<String> children = getFragmentsPerContig(destContigFolder);
+        if (children == null)
+        {
+            ctx.getLogger().warn("expected folder not found: " + destContigFolder.getPath());
         }
         else
         {
-            ctx.getLogger().warn("expected folder not found: " + destContigFolder.getPath());
+            ctx.getLogger().info(contigName + " total fragments: " + children.size());
         }
     }
 
+    private List<String> getFragmentsPerContig(File destContigFolder)
+    {
+        if (destContigFolder.exists())
+        {
+            List<String> children = Arrays.stream(destContigFolder.listFiles(x -> {
+                return  x.isDirectory() && !"genomicsdb_meta_dir".equals(x.getName());
+            })).map(File::getName).collect(Collectors.toList());
+
+            Collections.sort(children);
+
+            return children;
+        }
+
+        return null;
+    }
+
     private boolean doCopyLocal(JSONObject params)
     {
         return params.optBoolean("doCopyGVcfLocal", false);
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/GenomicsDbImportWrapper.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/GenomicsDbImportWrapper.java
@@ -88,6 +88,9 @@ public void execute(ReferenceGenome genome, List<File> inputGvcfs, File outputFi
             args.add(intervalList.getPath());
         }
 
+        args.add("-R");
+        args.add(genome.getWorkingFastaFile().getPath());
+
         execute(args);
 
         if (!outputFile.exists())
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/ChainFileValidator.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/ChainFileValidator.java
@@ -7,11 +7,19 @@
 import org.junit.Assert;
 import org.junit.Test;
 import org.labkey.api.collections.CaseInsensitiveHashMap;
+import org.labkey.api.data.CompareType;
 import org.labkey.api.data.DbScope;
+import org.labkey.api.data.Results;
 import org.labkey.api.data.SQLFragment;
 import org.labkey.api.data.Selector;
+import org.labkey.api.data.SimpleFilter;
 import org.labkey.api.data.SqlSelector;
+import org.labkey.api.data.TableSelector;
+import org.labkey.api.query.FieldKey;
+import org.labkey.api.sequenceanalysis.RefNtSequenceModel;
 import org.labkey.api.util.FileUtil;
+import org.labkey.api.util.PageFlowUtil;
+import org.labkey.sequenceanalysis.SequenceAnalysisSchema;
 
 import java.io.BufferedWriter;
 import java.io.File;
@@ -301,6 +309,23 @@ public void exec(ResultSet rs) throws SQLException
                 }
             });
 
+            SimpleFilter filter = new SimpleFilter(FieldKey.fromString("library_id"), genomeId);
+            filter.addCondition(FieldKey.fromString("alias"), null, CompareType.NONBLANK);
+            TableSelector ts = new TableSelector(SequenceAnalysisSchema.getTable(SequenceAnalysisSchema.TABLE_REF_LIBRARY_MEMBERS), PageFlowUtil.set("ref_nt_id", "alias"), filter, null);
+            if (ts.exists())
+            {
+                ts.forEachResults(new Selector.ForEachBlock<Results>()
+               {
+                   @Override
+                   public void exec(Results rs) throws SQLException
+                   {
+                       String name = RefNtSequenceModel.getForRowId(rs.getInt(FieldKey.fromString("ref_nt_id"))).getName();
+                       String alias = rs.getString(FieldKey.fromString("alias"));
+                       cachedReferences.put(alias, name);
+                   }
+               });
+            }
+
             _cachedReferencesByGenome.put(genomeId, cachedReferences);
         }
 
diff --git a/singlecell/resources/chunks/Downsample.R b/singlecell/resources/chunks/Downsample.R
@@ -2,7 +2,7 @@ for (datasetId in names(seuratObjects)) {
     seuratObj <- seuratObjects[[datasetId]]
     seuratObjects[[datasetId]] <- NULL
 
-    CellMembrane::DownsampleSeurat(seuratObj, targetCells = targetCells, subsetFields = subsetFields, seed = seed)
+    seuratObj <- CellMembrane::DownsampleSeurat(seuratObj, targetCells = targetCells, subsetFields = subsetFields, seed = seed)
 
     newSeuratObjects[[datasetId]] <- seuratObj
 
diff --git a/singlecell/resources/chunks/SubsetSeurat.R b/singlecell/resources/chunks/SubsetSeurat.R
@@ -2,7 +2,7 @@ for (datasetId in names(seuratObjects)) {
     seuratObj <- seuratObjects[[datasetId]]
     seuratObjects[[datasetId]] <- NULL
 
-    seuratObj <- CellMembrane::SubsetSeurat(seuratObj, expressionStrings = expressionStrings)
+    <SUBSETS>
 
     newSeuratObjects[[datasetId]] <- seuratObj
 
diff --git a/singlecell/resources/views/singleCellProcessing.html b/singlecell/resources/views/singleCellProcessing.html
@@ -3,7 +3,8 @@
         var webpart = <%=webpartContext%>;
 
         Ext4.create('SingleCell.panel.SingleCellProcessingPanel', {
-            outputFileIds: LABKEY.ActionURL.getParameter("outputFileIds") ? LABKEY.ActionURL.getParameter("outputFileIds").split(';') : []
+            outputFileIds: LABKEY.ActionURL.getParameter("outputFileIds") ? LABKEY.ActionURL.getParameter("outputFileIds").split(';') : [],
+            handlerClass: LABKEY.ActionURL.getParameter("handlerClass") || 'ProcessSingleCellHandler'
         }).render(webpart.wrapperDivId);
     });
 
diff --git a/singlecell/resources/web/singlecell/panel/SingleCellProcessingPanel.js b/singlecell/resources/web/singlecell/panel/SingleCellProcessingPanel.js
@@ -1,6 +1,7 @@
 Ext4.define('SingleCell.panel.SingleCellProcessingPanel', {
     extend: 'SequenceAnalysis.panel.BaseSequencePanel',
     alias: 'widget.singlecell-singlecellprocessingpanel',
+    handlerClass: 'ProcessSingleCellHandler',
 
     jobType: 'singleCell',
 
@@ -250,7 +251,7 @@ Ext4.define('SingleCell.panel.SingleCellProcessingPanel', {
 
         Ext4.Msg.wait('Submitting...');
         var json = {
-            handlerClass: 'org.labkey.singlecell.analysis.ProcessSingleCellHandler',
+            handlerClass: 'org.labkey.singlecell.analysis.' + this.handlerClass,
             outputFileIds: this.outputFileIds,
             params: Ext4.encode(values)
         };
diff --git a/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java b/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java
@@ -68,12 +68,6 @@ public String getDescription()
         return "Run one or more tools to process 10x scRNA-seq Data";
     }
 
-    @Override
-    public ActionURL getButtonSuccessUrl(Container c, User u, List<Integer> outputFileIds)
-    {
-        return DetailsURL.fromString("/singlecell/singleCellProcessing.view?outputFileIds=" + StringUtils.join(outputFileIds, ";"), c).getActionURL();
-    }
-
     @Override
     public String getButtonJSHandler()
     {
diff --git a/singlecell/src/org/labkey/singlecell/analysis/ProcessSeuratObjectHandler.java b/singlecell/src/org/labkey/singlecell/analysis/ProcessSeuratObjectHandler.java
@@ -27,6 +27,12 @@ public boolean canProcess(SequenceOutputFile f)
         return f.getFile() != null && SEURAT_TYPE.isType(f.getFile());
     }
 
+    @Override
+    public ActionURL getButtonSuccessUrl(Container c, User u, List<Integer> outputFileIds)
+    {
+        return DetailsURL.fromString("/singlecell/singleCellProcessing.view?handlerClass=ProcessSeuratObjectHandler&outputFileIds=" + StringUtils.join(outputFileIds, ";"), c).getActionURL();
+    }
+
     @Override
     public SequenceOutputProcessor getProcessor()
     {
diff --git a/singlecell/src/org/labkey/singlecell/analysis/ProcessSingleCellHandler.java b/singlecell/src/org/labkey/singlecell/analysis/ProcessSingleCellHandler.java
@@ -26,6 +26,12 @@ public boolean canProcess(SequenceOutputFile f)
         return f.getFile() != null && LOUPE_TYPE.isType(f.getFile());
     }
 
+    @Override
+    public ActionURL getButtonSuccessUrl(Container c, User u, List<Integer> outputFileIds)
+    {
+        return DetailsURL.fromString("/singlecell/singleCellProcessing.view?handlerClass=ProcessSingleCellHandler&outputFileIds=" + StringUtils.join(outputFileIds, ";"), c).getActionURL();
+    }
+
     @Override
     public SequenceOutputProcessor getProcessor()
     {
diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/FilterRawCounts.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/FilterRawCounts.java
@@ -35,11 +35,13 @@ public Provider()
                     SeuratToolParameter.create("pMitoLow", "Min Percent Mito", "Cells percent mitochondrial genes below this value will be discarded", "ldk-numberfield", new JSONObject(){{
                         put("minValue", 0);
                         put("maxValue", 1);
+                        put("decimalPrecision", 6);
                     }}, 0, "pMito.low", false),
                     SeuratToolParameter.create("pMitoHigh", "Max Percent Mito", "Cells percent mitochondrial genes above this value will be discarded", "ldk-numberfield", new JSONObject(){{
                         put("minValue", 0);
                         put("maxValue", 1);
-                    }}, 0.15, "pMito.high", false)
+                        put("decimalPrecision", 6);
+                    }}, 0.10, "pMito.high", false)
                     ), null, null);
         }
 
diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/NormalizeAndScale.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/NormalizeAndScale.java
@@ -36,7 +36,7 @@ public Provider()
                     SeuratToolParameter.create("featuresToRegress", "Features to Regress", "These features, entered comma-separated or one/line, will be passed to Seurat::ScaleData vars.to.regress", "sequenceanalysis-trimmingtextarea", new JSONObject(){{
                         put("height", 150);
                         put("delimiter", ",");
-                    }}, "nCount_RNA,p.mito"),
+                    }}, null),
                     SeuratToolParameter.create("scaleVariableFeaturesOnly", "ScaleData On Variable Features Only", "If checked, ScaleData will only be performed on VariableFeatures, which should dramatically reduce time and memory", "checkbox", new JSONObject(){{
                         put("checked", true);
                     }}, true),
diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+ALTER TABLE sequenceanalysis.reference_library_members ADD alias VARCHAR(4000);`
Original file line number	Diff line number	Diff line change
`@@ -178,7 +178,7 @@ public String getName()`
`178`	`178`	`@Override`
`179`	`179`	`public Double getSchemaVersion()`
`180`	`180`	`{`
`181`		`- return 12.324;`
	`181`	`+ return 12.325;`
`182`	`182`	`}`
`183`	`183`
`184`	`184`	`@Override`
Original file line number	Diff line number	Diff line change
`@@ -357,6 +357,8 @@ private Map<String, Pair<String, Integer>> getNameTranslationMap(List<RefNtSeque`
`357`	`357`
`358`	`358`	`getJob().getLogger().info("building list of chromosome name translations");`
`359`	`359`	`Map<String, Pair<String, Integer>> ret = new CaseInsensitiveHashMap<>();`
	`360`	`+`
	`361`	`+ TableInfo rlm = SequenceAnalysisSchema.getTable(SequenceAnalysisSchema.TABLE_REF_LIBRARY_MEMBERS);`
`360`	`362`	`for (RefNtSequenceModel m : refNtSequenceModels)`
`361`	`363`	`{`
`362`	`364`	`if (m.getOffsetsFile() != null && m.getOffsetsFile().exists())`
`@@ -446,6 +448,18 @@ private Map<String, Pair<String, Integer>> getNameTranslationMap(List<RefNtSeque`
`446`	`448`	`String strPart = m.getName().toLowerCase().replaceAll("^chr", "");`
`447`	`449`	`ret.put(strPart, Pair.of(m.getName(), 0));`
`448`	`450`	`}`
	`451`	`+`
	`452`	`+ SimpleFilter filter = new SimpleFilter(FieldKey.fromString("library_id"), getPipelineJob().getLibraryId());`
	`453`	`+ filter.addCondition(FieldKey.fromString("ref_nt_id"), m.getRowid());`
	`454`	`+ TableSelector ts = new TableSelector(rlm, PageFlowUtil.set("alias"), filter, null);`
	`455`	`+ if (ts.exists())`
	`456`	`+ {`
	`457`	`+ String alias = ts.getObject(String.class);`
	`458`	`+ if (alias != null)`
	`459`	`+ {`
	`460`	`+ ret.put(alias, Pair.of(m.getName(), 0));`
	`461`	`+ }`
	`462`	`+ }`
`449`	`463`	`}`
`450`	`464`	`}`
`451`	`465`	`}`
Original file line number	Diff line number	Diff line change
`@@ -40,12 +40,14 @@`
`40`	`40`	`import java.util.ArrayList;`
`41`	`41`	`import java.util.Arrays;`
`42`	`42`	`import java.util.Collection;`
	`43`	`+import java.util.Collections;`
`43`	`44`	`import java.util.Date;`
`44`	`45`	`import java.util.HashSet;`
`45`	`46`	`import java.util.LinkedHashSet;`
`46`	`47`	`import java.util.List;`
`47`	`48`	`import java.util.Map;`
`48`	`49`	`import java.util.Set;`
	`50`	`+import java.util.stream.Collectors;`
`49`	`51`
`50`	`52`	`abstract public class AbstractGenomicsDBImportHandler extends AbstractParameterizedOutputHandler<SequenceOutputHandler.SequenceOutputProcessor> implements SequenceOutputHandler.TracksVCF, SequenceOutputHandler.HasCustomVariantMerge, VariantProcessingStep.MayRequirePrepareTask, VariantProcessingStep.SupportsScatterGather`
`51`	`53`	`{`
`@@ -302,16 +304,28 @@ Set<String> getContigsInInputs(List<File> inputVCFs, Logger log) throws Pipeline`
`302`	`304`	`return _contigsInInputs;`
`303`	`305`	`}`
`304`	`306`
`305`		`- private void copyToLevelFiles(PipelineJob job, File sourceWorkspace, File destinationWorkspace, boolean overwrite) throws IOException`
	`307`	`+ private void copyToLevelFiles(PipelineJob job, File sourceWorkspace, File destinationWorkspace, boolean removeOtherFiles, boolean overwriteExisting) throws IOException`
`306`	`308`	`{`
`307`	`309`	`job.getLogger().info("Copying top-level files from: " + sourceWorkspace.getPath());`
	`310`	`+ if (removeOtherFiles)`
	`311`	`+ {`
	`312`	`+ for (File f : destinationWorkspace.listFiles())`
	`313`	`+ {`
	`314`	`+ if (!f.isDirectory())`
	`315`	`+ {`
	`316`	`+ job.getLogger().debug("deleting existing top-level file: " + f.getPath());`
	`317`	`+ f.delete();`
	`318`	`+ }`
	`319`	`+ }`
	`320`	`+ }`
	`321`	`+`
`308`	`322`	`for (String fn : Arrays.asList("callset.json", "vidmap.json", "vcfheader.vcf", "__tiledb_workspace.tdb"))`
`309`	`323`	`{`
`310`	`324`	`File source = new File(sourceWorkspace, fn);`
`311`	`325`	`File dest = new File(destinationWorkspace, fn);`
`312`	`326`	`if (dest.exists())`
`313`	`327`	`{`
`314`		`- if (!overwrite)`
	`328`	`+ if (!overwriteExisting)`
`315`	`329`	`{`
`316`	`330`	`job.getLogger().debug("workspace file exists, will not overwrite: " + dest.getPath());`
`317`	`331`	`continue;`
`@@ -457,7 +471,7 @@ else if (genomeIds.isEmpty())`
`457`	`471`	`}`
`458`	`472`
`459`	`473`	`File sourceWorkspace = getSourceWorkspace(ctx.getParams(), ctx.getSequenceSupport());`
`460`		`- copyWorkspace(ctx, sourceWorkspace, workingDestinationWorkspaceFolder, genome, toDelete, !genomicsDbCompleted);`
	`474`	`+ copyWorkspace(ctx, sourceWorkspace, workingDestinationWorkspaceFolder, genome, toDelete, !genomicsDbCompleted, !genomicsDbCompleted, !genomicsDbCompleted);`
`461`	`475`	`}`
`462`	`476`	`else`
`463`	`477`	`{`
`@@ -587,7 +601,7 @@ else if (genomeIds.isEmpty())`
`587`	`601`
`588`	`602`	`if (!copyToSourceDone.exists())`
`589`	`603`	`{`
`590`		`- copyWorkspace(ctx, workingDestinationWorkspaceFolder, workspaceLocalDir, genome, toDelete, true);`
	`604`	`+ copyWorkspace(ctx, workingDestinationWorkspaceFolder, workspaceLocalDir, genome, toDelete, true, false, false);`
`591`	`605`
`592`	`606`	`try`
`593`	`607`	`{`
`@@ -635,7 +649,7 @@ else if (genomeIds.isEmpty())`
`635`	`649`	`}`
`636`	`650`	`}`
`637`	`651`
`638`		`- private void copyWorkspace(JobContext ctx, File sourceWorkspace, File destinationWorkspaceFolder, ReferenceGenome genome, Collection<File> toDelete, boolean alwaysPerformRsync) throws PipelineJobException`
	`652`	`+ private void copyWorkspace(JobContext ctx, File sourceWorkspace, File destinationWorkspaceFolder, ReferenceGenome genome, Collection<File> toDelete, boolean alwaysPerformRsync, boolean overwriteTopLevelFiles, boolean removeExistingTopLevelFiles) throws PipelineJobException`
`639`	`653`	`{`
`640`	`654`	`if (!destinationWorkspaceFolder.exists())`
`641`	`655`	`{`
`@@ -665,14 +679,15 @@ private void copyWorkspace(JobContext ctx, File sourceWorkspace, File destinatio`
`665`	`679`	`else`
`666`	`680`	`{`
`667`	`681`	`ctx.getLogger().info("has been copied, skipping: " + i.getContig());`
	`682`	`+ assertContigFoldersEqual(sourceFolder, destContigFolder);`
`668`	`683`	`reportFragmentsPerContig(ctx, destContigFolder, i.getContig());`
`669`	`684`	`continue;`
`670`	`685`	`}`
`671`	`686`	`}`
`672`	`687`
`673`	`688`	`if (!haveCopiedTopLevelFiles)`
`674`	`689`	`{`
`675`		`- copyToLevelFiles(ctx.getJob(), sourceWorkspace, destinationWorkspaceFolder, false);`
	`690`	`+ copyToLevelFiles(ctx.getJob(), sourceWorkspace, destinationWorkspaceFolder, removeExistingTopLevelFiles, overwriteTopLevelFiles);`
`676`	`691`	`haveCopiedTopLevelFiles = true;`
`677`	`692`	`}`
`678`	`693`
`@@ -704,6 +719,7 @@ private void copyWorkspace(JobContext ctx, File sourceWorkspace, File destinatio`
`704`	`719`	`}`
`705`	`720`
`706`	`721`	`FileUtils.touch(copyDone);`
	`722`	`+ assertContigFoldersEqual(sourceFolder, destContigFolder);`
`707`	`723`	`reportFragmentsPerContig(ctx, destContigFolder, i.getContig());`
`708`	`724`	`}`
`709`	`725`	`catch (IOException e)`
`@@ -718,22 +734,46 @@ private File getCopyToSourceDone(JobContext ctx)`
`718`	`734`	`return new File(ctx.getSourceDirectory(), "copyToWebserver.done");`
`719`	`735`	`}`
`720`	`736`
`721`		`- private void reportFragmentsPerContig(JobContext ctx, File destContigFolder, String contigName)`
	`737`	`+ private void assertContigFoldersEqual(File sourceFolder, File destContigFolder) throws IllegalArgumentException`
`722`	`738`	`{`
`723`		`- if (destContigFolder.exists())`
	`739`	`+ List<String> sourceFiles = getFragmentsPerContig(sourceFolder);`
	`740`	`+ List<String> destFiles = getFragmentsPerContig(destContigFolder);`
	`741`	`+`
	`742`	`+ if (!sourceFiles.equals(destFiles))`
`724`	`743`	`{`
`725`		`- File[] children = destContigFolder.listFiles(x -> {`
`726`		`- return x.isDirectory() && !"genomicsdb_meta_dir".equals(x.getName());`
`727`		`- });`
	`744`	`+ throw new IllegalArgumentException("Source and destination contig files not equal for: " + destContigFolder.getPath());`
	`745`	`+ }`
	`746`	`+ }`
`728`	`747`
`729`		`- ctx.getLogger().info(contigName + " total fragments: " + children.length);`
	`748`	`+ private void reportFragmentsPerContig(JobContext ctx, File destContigFolder, String contigName)`
	`749`	`+ {`
	`750`	`+ List<String> children = getFragmentsPerContig(destContigFolder);`
	`751`	`+ if (children == null)`
	`752`	`+ {`
	`753`	`+ ctx.getLogger().warn("expected folder not found: " + destContigFolder.getPath());`
`730`	`754`	`}`
`731`	`755`	`else`
`732`	`756`	`{`
`733`		`- ctx.getLogger().warn("expected folder not found: " + destContigFolder.getPath());`
	`757`	`+ ctx.getLogger().info(contigName + " total fragments: " + children.size());`
`734`	`758`	`}`
`735`	`759`	`}`
`736`	`760`
	`761`	`+ private List<String> getFragmentsPerContig(File destContigFolder)`
	`762`	`+ {`
	`763`	`+ if (destContigFolder.exists())`
	`764`	`+ {`
	`765`	`+ List<String> children = Arrays.stream(destContigFolder.listFiles(x -> {`
	`766`	`+ return x.isDirectory() && !"genomicsdb_meta_dir".equals(x.getName());`
	`767`	`+ })).map(File::getName).collect(Collectors.toList());`
	`768`	`+`
	`769`	`+ Collections.sort(children);`
	`770`	`+`
	`771`	`+ return children;`
	`772`	`+ }`
	`773`	`+`
	`774`	`+ return null;`
	`775`	`+ }`
	`776`	`+`
`737`	`777`	`private boolean doCopyLocal(JSONObject params)`
`738`	`778`	`{`
`739`	`779`	`return params.optBoolean("doCopyGVcfLocal", false);`
Original file line number	Diff line number	Diff line change
`@@ -88,6 +88,9 @@ public void execute(ReferenceGenome genome, List<File> inputGvcfs, File outputFi`
`88`	`88`	`args.add(intervalList.getPath());`
`89`	`89`	`}`
`90`	`90`
	`91`	`+ args.add("-R");`
	`92`	`+ args.add(genome.getWorkingFastaFile().getPath());`
	`93`	`+`
`91`	`94`	`execute(args);`
`92`	`95`
`93`	`96`	`if (!outputFile.exists())`