Merge pull request #69 from LabKey/fb_merge_discvr-20.7

bbimber · web-flow · commit 5579edf83aff · 2020-10-29T12:41:25.000-07:00
Merge discvr-20.7 to develop
diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AlignerIndexUtil.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AlignerIndexUtil.java
@@ -49,9 +49,14 @@ public static boolean copyIndexIfExists(PipelineContext ctx, AlignmentOutputImpl
         return verifyOrCreateCachedIndex(ctx, ctx.getWorkDir(), output, localName, webserverName, genome, forceCopyLocal);
     }
 
-    public static File getWebserverIndexDir(ReferenceGenome genome, String name)
+    public static File getIndexDir(ReferenceGenome genome, String name)
     {
-        return new File(genome.getSourceFastaFile().getParentFile(), (genome.isTemporaryGenome() ? "" : INDEX_DIR + "/") + name);
+        return getIndexDir(genome, name, false);
+    }
+
+    public static File getIndexDir(ReferenceGenome genome, String name, boolean useWebserverDir)
+    {
+        return new File(useWebserverDir ? genome.getSourceFastaFile().getParentFile() : genome.getWorkingFastaFile().getParentFile(), (genome.isTemporaryGenome() ? "" : INDEX_DIR + "/") + name);
     }
 
     /**
@@ -63,7 +68,7 @@ private static boolean verifyOrCreateCachedIndex(PipelineContext ctx, @Nullable
         if (genome != null)
         {
             //NOTE: when we cache the indexes with the source FASTA genome, we store all aligners under the folder /alignerIndexes.  When these are temporary genomes, they're top-level
-            File webserverIndexDir = getWebserverIndexDir(genome, webserverName);
+            File webserverIndexDir = getIndexDir(genome, webserverName, true);
             if (webserverIndexDir.exists())
             {
                 ctx.getLogger().info("previously created index found, no need to recreate");
@@ -180,7 +185,7 @@ public static void saveCachedIndex(boolean hasCachedIndex, PipelineContext ctx,
 
                 lockFile.delete();
 
-                ReferenceGenomeManager.get().markGenomeModified(genome);
+                ReferenceGenomeManager.get().markGenomeModified(genome, ctx.getLogger());
             }
             catch (IOException e)
             {
diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/ReferenceGenomeManager.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/ReferenceGenomeManager.java
@@ -1,6 +1,6 @@
 package org.labkey.api.sequenceanalysis.pipeline;
 
-import org.apache.commons.io.FileUtils;
+import com.google.common.io.Files;
 import org.apache.logging.log4j.Logger;
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper;
@@ -57,13 +57,14 @@ private boolean isUpToDate(ReferenceGenome genome)
         long lastUpdated = localFile.lastModified();
         long lastSync = remoteFile.lastModified();
 
-        return lastUpdated >= lastSync;
+        return lastSync >= lastUpdated;
     }
 
-    public void markGenomeModified(ReferenceGenome genome) throws IOException
+    public void markGenomeModified(ReferenceGenome genome, Logger log) throws IOException
     {
         File toUpdate = getLocalUpdateFile(genome);
-        FileUtils.touch(toUpdate);
+        log.info("Marking genome as modified: " + toUpdate.getPath());
+        Files.touch(toUpdate);
     }
 
     public void cacheGenomeLocally(ReferenceGenome genome, Logger log) throws PipelineJobException
@@ -100,10 +101,10 @@ public void cacheGenomeLocally(ReferenceGenome genome, Logger log) throws Pipeli
             File lastUpdate = getLocalUpdateFile(genome);
             if (!lastUpdate.exists())
             {
-                FileUtils.touch(lastUpdate);
+                Files.touch(lastUpdate);
             }
 
-            FileUtils.touch(getRemoteSyncFile(genome.getGenomeId()));
+            Files.touch(getRemoteSyncFile(genome.getGenomeId()));
         }
         catch (IOException e)
         {
diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequenceOutputHandler.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequenceOutputHandler.java
@@ -247,6 +247,11 @@ public static interface TracksVCF
     {
         public File getScatterJobOutput(JobContext ctx) throws PipelineJobException;
 
+        default File finalizeScatterJobOutput(JobContext ctx, File primaryOutput) throws PipelineJobException
+        {
+            return primaryOutput;
+        }
+
         public SequenceOutputFile createFinalSequenceOutput(PipelineJob job, File processed, List<SequenceOutputFile> inputFiles) throws PipelineJobException;
     }
 
diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequencePipelineService.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequencePipelineService.java
@@ -127,5 +127,5 @@ static public void setInstance(SequencePipelineService instance)
 
     abstract public PreprocessingStep.Output simpleTrimFastqPair(File fq1, File fq2, List<String> params, File outDir, Logger log) throws PipelineJobException;
 
-    abstract public File runCiteSeqCount(PipelineStepOutput output, @Nullable String outputCategory, Readset htoReadset, File htoList, File cellBarcodeList, File outputDir, String basename, Logger log, List<String> extraArgs, boolean doHtoFiltering, @Nullable Integer minCountPerCell, File localPipelineDir, @Nullable Integer editDistance, boolean scanEditDistances, Readset parentReadset, @Nullable Integer genomeId, boolean generateHtoCalls, boolean createOutputFiles) throws PipelineJobException;
+    abstract public File runCiteSeqCount(PipelineStepOutput output, @Nullable String outputCategory, Readset htoReadset, File htoList, File cellBarcodeList, File outputDir, String basename, Logger log, List<String> extraArgs, boolean doHtoFiltering, @Nullable Integer minCountPerCell, File localPipelineDir, @Nullable Integer editDistance, boolean scanEditDistances, Readset parentReadset, @Nullable Integer genomeId, boolean generateHtoCalls, boolean createOutputFiles, boolean useSeurat, boolean useMultiSeq) throws PipelineJobException;
 }
diff --git a/SequenceAnalysis/resources/external/scRNAseq/htoClassifier.Rmd b/SequenceAnalysis/resources/external/scRNAseq/htoClassifier.Rmd
@@ -41,7 +41,7 @@ if (nrow(barcodeData) > 0 && ncol(barcodeData) > 0){
 ```{r GenerateCalls, fig.width=12}
 
 if (nrow(barcodeData) > 0 && ncol(barcodeData) > 0){
-    dt <- GenerateCellHashingCalls(barcodeData = barcodeData, outFile = finalCallFile, allCallsOutFile = allCallsOutFile)
+    dt <- GenerateCellHashingCalls(barcodeData = barcodeData, outFile = finalCallFile, allCallsOutFile = allCallsOutFile, useSeurat = useSeurat, useMultiSeq = useMultiSeq)
 
     if (exists('whitelistFile') && !is.null(whitelistFile)){
       GenerateSummaryForExpectedBarcodes(dt, whitelistFile=whitelistFile, outputFile=metricsFile, barcodeData=barcodeData)
diff --git a/SequenceAnalysis/resources/external/scRNAseq/htoClassifier.sh b/SequenceAnalysis/resources/external/scRNAseq/htoClassifier.sh
@@ -14,9 +14,12 @@ RAW_CALLS=$4
 DO_HTO_FILTER=$5
 MIN_READS_PER_CELL=$6
 METRICS_FILE=$7
+USE_SEURAT=$8
+USE_MULTISEQ=$9
+
 WHITELIST="whitelistFile<-NULL;"
-if [ $# -ge 8 ];then
-    WHITELIST="whitelistFile<-'"${8}"';"
+if [ $# -ge 10 ];then
+    WHITELIST="whitelistFile<-'"${10}"';"
 fi
 
 ENV_OPTS=""
@@ -33,4 +36,4 @@ fi
 
 sudo $DOCKER pull bimberlab/oosap
 
-sudo $DOCKER run --rm=true $RAM_OPTS $ENV_OPTS -v "${WD}:/work" -v "${HOME}:/homeDir" -u $UID -e USERID=$UID -w /work -e HOME=/homeDir bimberlab/oosap Rscript -e "barcodeDir <- '"${CITESEQ_COUNT_DIR}"';finalCallFile <- '"${FINAL_CALLS}"';doHtoFilter <- "${DO_HTO_FILTER}";maxValueForColSumFilter <- "${MIN_READS_PER_CELL}";allCallsOutFile <- '"${RAW_CALLS}"';metricsFile <- '"${METRICS_FILE}"';"${WHITELIST}"rmarkdown::render('htoClassifier.Rmd', output_file = '"${HTML_FILE}"')"
+sudo $DOCKER run --rm=true $RAM_OPTS $ENV_OPTS -v "${WD}:/work" -v "${HOME}:/homeDir" -u $UID -e USERID=$UID -w /work -e HOME=/homeDir bimberlab/oosap Rscript -e "barcodeDir <- '"${CITESEQ_COUNT_DIR}"';finalCallFile <- '"${FINAL_CALLS}"';doHtoFilter <- "${DO_HTO_FILTER}";maxValueForColSumFilter <- "${MIN_READS_PER_CELL}";allCallsOutFile <- '"${RAW_CALLS}"';metricsFile <- '"${METRICS_FILE}"';useSeurat <- ${USE_SEURAT};useMultiSeq <- ${USE_MULTISEQ};"${WHITELIST}"rmarkdown::render('htoClassifier.Rmd', output_file = '"${HTML_FILE}"')"
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java
@@ -528,10 +528,10 @@ else if (!SequenceUtil.hasLineCount(f))
     }
 
     @Override
-    public File runCiteSeqCount(PipelineStepOutput output, @Nullable String outputCategory, Readset htoReadset, File htoList, File cellBarcodeList, File outputDir, String basename, Logger log, List<String> extraArgs, boolean doHtoFiltering, @Nullable Integer minCountPerCell, File localPipelineDir, @Nullable Integer editDistance, boolean scanEditDistances, Readset parentReadset, @Nullable Integer genomeId, boolean generateHtoCalls, boolean createOutputFiles) throws PipelineJobException
+    public File runCiteSeqCount(PipelineStepOutput output, @Nullable String outputCategory, Readset htoReadset, File htoList, File cellBarcodeList, File outputDir, String basename, Logger log, List<String> extraArgs, boolean doHtoFiltering, @Nullable Integer minCountPerCell, File localPipelineDir, @Nullable Integer editDistance, boolean scanEditDistances, Readset parentReadset, @Nullable Integer genomeId, boolean generateHtoCalls, boolean createOutputFiles, boolean useSeurat, boolean useMultiSeq) throws PipelineJobException
     {
         CellHashingHandler handler = new CellHashingHandler();
 
-        return handler.runCiteSeqCount(output, outputCategory, htoReadset, htoList, cellBarcodeList, outputDir, basename, log, extraArgs, doHtoFiltering, minCountPerCell, localPipelineDir, editDistance, scanEditDistances, parentReadset, genomeId, generateHtoCalls ? CellHashingHandler.BARCODE_TYPE.hashing : CellHashingHandler.BARCODE_TYPE.citeseq, createOutputFiles);
+        return handler.runCiteSeqCount(output, outputCategory, htoReadset, htoList, cellBarcodeList, outputDir, basename, log, extraArgs, doHtoFiltering, minCountPerCell, localPipelineDir, editDistance, scanEditDistances, parentReadset, genomeId, generateHtoCalls ? CellHashingHandler.BARCODE_TYPE.hashing : CellHashingHandler.BARCODE_TYPE.citeseq, createOutputFiles, useSeurat, useMultiSeq);
     }
 }
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/CellHashingHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/CellHashingHandler.java
@@ -444,7 +444,7 @@ else if (line.startsWith("Percentage unmapped"))
         if (type.doGenerateCalls())
         {
             ctx.getJob().setStatus(PipelineJob.TaskStatus.running, "Generating HTO calls for edit distance: " + editDistance);
-            File htoCalls = generateFinalCalls(outputMatrix.getParentFile(), ctx.getOutputDir(), outputBasename, ctx.getLogger(), null, true, minCountPerCell, ctx.getSourceDirectory());
+            File htoCalls = generateFinalCalls(outputMatrix.getParentFile(), ctx.getOutputDir(), outputBasename, ctx.getLogger(), null, true, minCountPerCell, ctx.getSourceDirectory(), true, true);
             File html = new File(htoCalls.getParentFile(), outputBasename + ".html");
 
             if (!html.exists())
@@ -665,7 +665,7 @@ private File ensureLocalCopy(File input, File outputDir, Logger log, Set<File> t
         return input;
     }
 
-    public File generateFinalCalls(File citeSeqCountOutDir, File outputDir, String basename, Logger log, @Nullable File cellBarcodeWhitelist, boolean doHtoFiltering, Integer minCountPerCell, File localPipelineDir) throws PipelineJobException
+    private File generateFinalCalls(File citeSeqCountOutDir, File outputDir, String basename, Logger log, @Nullable File cellBarcodeWhitelist, boolean doHtoFiltering, Integer minCountPerCell, File localPipelineDir, boolean useSeurat, boolean useMultiSeq) throws PipelineJobException
     {
         log.debug("generating final calls from folder: " + citeSeqCountOutDir.getPath());
 
@@ -695,6 +695,9 @@ public File generateFinalCalls(File citeSeqCountOutDir, File outputDir, String b
         File rawCallsFile = new File(outputDir, basename + ".raw.txt");
         File metricsFile = getMetricsFile(callsFile);
         List<String> args = new ArrayList<>(Arrays.asList("/bin/bash", scriptWrapper, citeSeqCountOutDir.getName(), htmlFile.getName(), callsFile.getName(), rawCallsFile.getName(), (doHtoFiltering ? "T" : "F"), (minCountPerCell == null ? "0" : minCountPerCell.toString()), metricsFile.getName()));
+        args.add(useSeurat ? "TRUE" : "FALSE");
+        args.add(useMultiSeq ? "TRUE" : "FALSE");
+
         if (cellBarcodeWhitelist != null)
         {
             args.add(cellBarcodeWhitelist.getName());
@@ -877,7 +880,7 @@ else if (htoReadset.getReadData().size() != 1)
         return ret;
     }
 
-    public File runCiteSeqCount(PipelineStepOutput output, String category, Readset htoReadset, File htoList, File cellBarcodeList, File outputDir, String basename, Logger log, List<String> extraArgs, boolean doHtoFiltering, @Nullable Integer minCountPerCell, File localPipelineDir, @Nullable Integer editDistance, boolean scanEditDistances, Readset parentReadset, @Nullable Integer genomeId, BARCODE_TYPE type, boolean createOutputFiles) throws PipelineJobException
+    public File runCiteSeqCount(PipelineStepOutput output, String category, Readset htoReadset, File htoList, File cellBarcodeList, File outputDir, String basename, Logger log, List<String> extraArgs, boolean doHtoFiltering, @Nullable Integer minCountPerCell, File localPipelineDir, @Nullable Integer editDistance, boolean scanEditDistances, Readset parentReadset, @Nullable Integer genomeId, BARCODE_TYPE type, boolean createOutputFiles, boolean useSeurat, boolean useMultiSeq) throws PipelineJobException
     {
         HtoMergeResult htoFastqs = possiblyMergeHtoFastqs(htoReadset, outputDir, log);
         if (!htoFastqs.intermediateFiles.isEmpty())
@@ -964,7 +967,7 @@ public File runCiteSeqCount(PipelineStepOutput output, String category, Readset
 
             File citeSeqCountOutDir = new File(outputDir, basename + ".citeSeqCounts." + ed + "." + type.name());
             String outputBasename = basename + "." + ed + "." + type.name();
-            Map<String, Object> callMap = executeCiteSeqCountWithJobCtx(outputDir, outputBasename, citeSeqCountOutDir, htoFastqs.files.getLeft(), htoFastqs.files.getRight(), toolArgs, ed, log, cellBarcodeList, doHtoFiltering, minCountPerCell, localPipelineDir, unknownBarcodeFile, type);
+            Map<String, Object> callMap = executeCiteSeqCountWithJobCtx(outputDir, outputBasename, citeSeqCountOutDir, htoFastqs.files.getLeft(), htoFastqs.files.getRight(), toolArgs, ed, log, cellBarcodeList, doHtoFiltering, minCountPerCell, localPipelineDir, unknownBarcodeFile, type, useSeurat, useMultiSeq);
             results.put(ed, callMap);
 
             if (type.doGenerateCalls())
@@ -1065,7 +1068,7 @@ public File runCiteSeqCount(PipelineStepOutput output, String category, Readset
         }
     }
 
-    private Map<String, Object> executeCiteSeqCountWithJobCtx(File outputDir, String basename, File citeSeqCountOutDir, File fastq1, File fastq2, List<String> baseArgs, Integer ed, Logger log, File cellBarcodeList, boolean doHtoFiltering, Integer minCountPerCell, File localPipelineDir, File unknownBarcodeFile, BARCODE_TYPE type) throws PipelineJobException
+    private Map<String, Object> executeCiteSeqCountWithJobCtx(File outputDir, String basename, File citeSeqCountOutDir, File fastq1, File fastq2, List<String> baseArgs, Integer ed, Logger log, File cellBarcodeList, boolean doHtoFiltering, Integer minCountPerCell, File localPipelineDir, File unknownBarcodeFile, BARCODE_TYPE type, boolean useSeurat, boolean useMultiSeq) throws PipelineJobException
     {
         CellHashingHandler.CiteSeqCountWrapper wrapper = new CellHashingHandler.CiteSeqCountWrapper(log);
         File doneFile = new File(citeSeqCountOutDir, "citeSeqCount." + type.name() + "." + ed + ".done");
@@ -1132,7 +1135,7 @@ else if (line.startsWith("Percentage unmapped"))
 
         if (type.doGenerateCalls())
         {
-            File htoCalls = generateFinalCalls(outputMatrix.getParentFile(), outputDir, basename, log, cellBarcodeList, doHtoFiltering, minCountPerCell, localPipelineDir);
+            File htoCalls = generateFinalCalls(outputMatrix.getParentFile(), outputDir, basename, log, cellBarcodeList, doHtoFiltering, minCountPerCell, localPipelineDir, useSeurat, useMultiSeq);
             if (!htoCalls.exists())
             {
                 throw new PipelineJobException("missing expected file: " + htoCalls.getPath());
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java
@@ -384,7 +384,7 @@ public RecordedActionSet run() throws PipelineJobException
             getJob().getLogger().info("creation complete");
 
             ReferenceGenome rg = SequenceAnalysisService.get().getReferenceGenome(rowId, getJob().getUser());
-            ReferenceGenomeManager.get().markGenomeModified(rg);
+            ReferenceGenomeManager.get().markGenomeModified(rg, getJob().getLogger());
 
             Set<GenomeTrigger> triggers = new HashSet<>(getPipelineJob().getExtraTriggers());
             if (getPipelineJob().isSkipTriggers())
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportGenomeTrackTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportGenomeTrackTask.java
@@ -143,7 +143,7 @@ public RecordedActionSet run() throws PipelineJobException
             }
 
             final int trackId = addTrackForLibrary(getPipelineJob().getTrack(), getPipelineJob().getTrackName(), getPipelineJob().getTrackDescription(), action);
-            ReferenceGenomeManager.get().markGenomeModified(SequenceAnalysisService.get().getReferenceGenome(getPipelineJob().getLibraryId(), getJob().getUser()));
+            ReferenceGenomeManager.get().markGenomeModified(SequenceAnalysisService.get().getReferenceGenome(getPipelineJob().getLibraryId(), getJob().getUser()), getJob().getLogger());
 
             Set<GenomeTrigger> triggers = SequenceAnalysisServiceImpl.get().getGenomeTriggers();
             if (!triggers.isEmpty())
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingRemoteSplitTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingRemoteSplitTask.java
@@ -110,10 +110,20 @@ private VariantProcessingJob getPipelineJob()
                 File output = ((SequenceOutputHandler.TracksVCF)handler).getScatterJobOutput(ctx);
                 try
                 {
-                    //NOTE: the VCF was copied back to the source dir, so translate paths
-                    String path = _wd.getRelativePath(output);
-                    output = new File(ctx.getSourceDirectory(), path);
-                    getPipelineJob().getScatterJobOutputs().put(getPipelineJob().getIntervalSetName(), output);
+                    output = ((SequenceOutputHandler.TracksVCF)handler).finalizeScatterJobOutput(ctx, output);
+
+                    // If the output is still under the work dir, translate path. Otherwise it was already copied to the the source dir
+                    if (output.getPath().startsWith(_wd.getDir().getPath()))
+                    {
+                        //NOTE: the VCF will be copied back to the source dir, so translate paths
+                        String path = _wd.getRelativePath(output);
+                        output = new File(ctx.getSourceDirectory(), path);
+                        getPipelineJob().getScatterJobOutputs().put(getPipelineJob().getIntervalSetName(), output);
+                    }
+                    else
+                    {
+                        ctx.getLogger().debug("Output has already been moved from workdir: " + output.getPath());
+                    }
                 }
                 catch (IOException e)
                 {
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/alignment/CellRangerWrapper.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/alignment/CellRangerWrapper.java
@@ -96,6 +96,9 @@ public Provider()
                         put("extensions", Arrays.asList("gtf"));
                         put("width", 400);
                         put("allowBlank", false);
+                    }}, null),
+                    ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("--chemistry"), "chemistry", "Chemistry", "This is usually left blank, in which case cellranger will auto-detect. Example values are: SC3Pv1, SC3Pv2, SC3Pv3, SC5P-PE, SC5P-R2, or SC5P-R1", "textfield", new JSONObject(){{
+
                     }}, null)
             ), PageFlowUtil.set("sequenceanalysis/field/GenomeFileSelectorField.js"), "https://support.10xgenomics.com/single-cell-gene-expression/software/pipelines/latest/what-is-cell-ranger", true, false, ALIGNMENT_MODE.MERGE_THEN_ALIGN);
         }
@@ -337,7 +340,7 @@ public AlignmentOutput performAlignment(Readset rs, File inputFastq1, @Nullable
             File gtfFile = getPipelineCtx().getSequenceSupport().getCachedData(gtfId);
             output.addInput(gtfFile, GTF_FILE);
 
-            File indexDir = AlignerIndexUtil.getWebserverIndexDir(referenceGenome, getIndexCachedDirName(getPipelineCtx().getJob()));
+            File indexDir = AlignerIndexUtil.getIndexDir(referenceGenome, getIndexCachedDirName(getPipelineCtx().getJob()));
             args.add("--transcriptome=" + indexDir.getPath());
 
             getWrapper().setWorkingDir(outputDirectory);
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/SNPEffStep.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/SNPEffStep.java
@@ -65,7 +65,7 @@ public static File checkOrCreateIndex(SequenceAnalysisJobSupport support, Logger
     {
         log.debug("checking for index");
 
-        File snpEffBaseDir = AlignerIndexUtil.getWebserverIndexDir(genome, NAME);
+        File snpEffBaseDir = AlignerIndexUtil.getIndexDir(genome, NAME);
         File geneFile = support.getCachedData(geneFileId);
         if (geneFile == null || !geneFile.exists())
         {
diff --git a/cluster/src/org/labkey/cluster/pipeline/AbstractClusterExecutionEngine.java b/cluster/src/org/labkey/cluster/pipeline/AbstractClusterExecutionEngine.java

Original file line number	Diff line number	Diff line change
`@@ -49,9 +49,14 @@ public static boolean copyIndexIfExists(PipelineContext ctx, AlignmentOutputImpl`
`49`	`49`	`return verifyOrCreateCachedIndex(ctx, ctx.getWorkDir(), output, localName, webserverName, genome, forceCopyLocal);`
`50`	`50`	`}`
`51`	`51`
`52`		`- public static File getWebserverIndexDir(ReferenceGenome genome, String name)`
	`52`	`+ public static File getIndexDir(ReferenceGenome genome, String name)`
`53`	`53`	`{`
`54`		`- return new File(genome.getSourceFastaFile().getParentFile(), (genome.isTemporaryGenome() ? "" : INDEX_DIR + "/") + name);`
	`54`	`+ return getIndexDir(genome, name, false);`
	`55`	`+ }`
	`56`	`+`
	`57`	`+ public static File getIndexDir(ReferenceGenome genome, String name, boolean useWebserverDir)`
	`58`	`+ {`
	`59`	`+ return new File(useWebserverDir ? genome.getSourceFastaFile().getParentFile() : genome.getWorkingFastaFile().getParentFile(), (genome.isTemporaryGenome() ? "" : INDEX_DIR + "/") + name);`
`55`	`60`	`}`
`56`	`61`
`57`	`62`	`/**`
`@@ -63,7 +68,7 @@ private static boolean verifyOrCreateCachedIndex(PipelineContext ctx, @Nullable`
`63`	`68`	`if (genome != null)`
`64`	`69`	`{`
`65`	`70`	`//NOTE: when we cache the indexes with the source FASTA genome, we store all aligners under the folder /alignerIndexes. When these are temporary genomes, they're top-level`
`66`		`- File webserverIndexDir = getWebserverIndexDir(genome, webserverName);`
	`71`	`+ File webserverIndexDir = getIndexDir(genome, webserverName, true);`
`67`	`72`	`if (webserverIndexDir.exists())`
`68`	`73`	`{`
`69`	`74`	`ctx.getLogger().info("previously created index found, no need to recreate");`
`@@ -180,7 +185,7 @@ public static void saveCachedIndex(boolean hasCachedIndex, PipelineContext ctx,`
`180`	`185`
`181`	`186`	`lockFile.delete();`
`182`	`187`
`183`		`- ReferenceGenomeManager.get().markGenomeModified(genome);`
	`188`	`+ ReferenceGenomeManager.get().markGenomeModified(genome, ctx.getLogger());`
`184`	`189`	`}`
`185`	`190`	`catch (IOException e)`
`186`	`191`	`{`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`package org.labkey.api.sequenceanalysis.pipeline;`
`2`	`2`
`3`		`-import org.apache.commons.io.FileUtils;`
	`3`	`+import com.google.common.io.Files;`
`4`	`4`	`import org.apache.logging.log4j.Logger;`
`5`	`5`	`import org.labkey.api.pipeline.PipelineJobException;`
`6`	`6`	`import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper;`
`@@ -57,13 +57,14 @@ private boolean isUpToDate(ReferenceGenome genome)`
`57`	`57`	`long lastUpdated = localFile.lastModified();`
`58`	`58`	`long lastSync = remoteFile.lastModified();`
`59`	`59`
`60`		`- return lastUpdated >= lastSync;`
	`60`	`+ return lastSync >= lastUpdated;`
`61`	`61`	`}`
`62`	`62`
`63`		`- public void markGenomeModified(ReferenceGenome genome) throws IOException`
	`63`	`+ public void markGenomeModified(ReferenceGenome genome, Logger log) throws IOException`
`64`	`64`	`{`
`65`	`65`	`File toUpdate = getLocalUpdateFile(genome);`
`66`		`- FileUtils.touch(toUpdate);`
	`66`	`+ log.info("Marking genome as modified: " + toUpdate.getPath());`
	`67`	`+ Files.touch(toUpdate);`
`67`	`68`	`}`
`68`	`69`
`69`	`70`	`public void cacheGenomeLocally(ReferenceGenome genome, Logger log) throws PipelineJobException`
`@@ -100,10 +101,10 @@ public void cacheGenomeLocally(ReferenceGenome genome, Logger log) throws Pipeli`
`100`	`101`	`File lastUpdate = getLocalUpdateFile(genome);`
`101`	`102`	`if (!lastUpdate.exists())`
`102`	`103`	`{`
`103`		`- FileUtils.touch(lastUpdate);`
	`104`	`+ Files.touch(lastUpdate);`
`104`	`105`	`}`
`105`	`106`
`106`		`- FileUtils.touch(getRemoteSyncFile(genome.getGenomeId()));`
	`107`	`+ Files.touch(getRemoteSyncFile(genome.getGenomeId()));`
`107`	`108`	`}`
`108`	`109`	`catch (IOException e)`
`109`	`110`	`{`
Original file line number	Diff line number	Diff line change
`@@ -247,6 +247,11 @@ public static interface TracksVCF`
`247`	`247`	`{`
`248`	`248`	`public File getScatterJobOutput(JobContext ctx) throws PipelineJobException;`
`249`	`249`
	`250`	`+ default File finalizeScatterJobOutput(JobContext ctx, File primaryOutput) throws PipelineJobException`
	`251`	`+ {`
	`252`	`+ return primaryOutput;`
	`253`	`+ }`
	`254`	`+`
`250`	`255`	`public SequenceOutputFile createFinalSequenceOutput(PipelineJob job, File processed, List<SequenceOutputFile> inputFiles) throws PipelineJobException;`
`251`	`256`	`}`
`252`	`257`
Original file line number	Diff line number	Diff line change
`@@ -127,5 +127,5 @@ static public void setInstance(SequencePipelineService instance)`
`127`	`127`
`128`	`128`	`abstract public PreprocessingStep.Output simpleTrimFastqPair(File fq1, File fq2, List<String> params, File outDir, Logger log) throws PipelineJobException;`
`129`	`129`
`130`		`- abstract public File runCiteSeqCount(PipelineStepOutput output, @Nullable String outputCategory, Readset htoReadset, File htoList, File cellBarcodeList, File outputDir, String basename, Logger log, List<String> extraArgs, boolean doHtoFiltering, @Nullable Integer minCountPerCell, File localPipelineDir, @Nullable Integer editDistance, boolean scanEditDistances, Readset parentReadset, @Nullable Integer genomeId, boolean generateHtoCalls, boolean createOutputFiles) throws PipelineJobException;`
	`130`	+ abstract public File runCiteSeqCount(PipelineStepOutput output, @Nullable String outputCategory, Readset htoReadset, File htoList, File cellBarcodeList, File outputDir, String basename, Logger log, List<String> extraArgs, boolean doHtoFiltering, @Nullable Integer minCountPerCell, File localPipelineDir, @Nullable Integer editDistance, boolean scanEditDistances, Readset parentReadset, @Nullable Integer genomeId, boolean generateHtoCalls, boolean createOutputFiles, boolean useSeurat, boolean useMultiSeq) throws PipelineJobException;
`131`	`131`	`}`
Original file line number	Diff line number	Diff line change
`@@ -528,10 +528,10 @@ else if (!SequenceUtil.hasLineCount(f))`
`528`	`528`	`}`
`529`	`529`
`530`	`530`	`@Override`
`531`		`- public File runCiteSeqCount(PipelineStepOutput output, @Nullable String outputCategory, Readset htoReadset, File htoList, File cellBarcodeList, File outputDir, String basename, Logger log, List<String> extraArgs, boolean doHtoFiltering, @Nullable Integer minCountPerCell, File localPipelineDir, @Nullable Integer editDistance, boolean scanEditDistances, Readset parentReadset, @Nullable Integer genomeId, boolean generateHtoCalls, boolean createOutputFiles) throws PipelineJobException`
	`531`	+ public File runCiteSeqCount(PipelineStepOutput output, @Nullable String outputCategory, Readset htoReadset, File htoList, File cellBarcodeList, File outputDir, String basename, Logger log, List<String> extraArgs, boolean doHtoFiltering, @Nullable Integer minCountPerCell, File localPipelineDir, @Nullable Integer editDistance, boolean scanEditDistances, Readset parentReadset, @Nullable Integer genomeId, boolean generateHtoCalls, boolean createOutputFiles, boolean useSeurat, boolean useMultiSeq) throws PipelineJobException
`532`	`532`	`{`
`533`	`533`	`CellHashingHandler handler = new CellHashingHandler();`
`534`	`534`
`535`		`- return handler.runCiteSeqCount(output, outputCategory, htoReadset, htoList, cellBarcodeList, outputDir, basename, log, extraArgs, doHtoFiltering, minCountPerCell, localPipelineDir, editDistance, scanEditDistances, parentReadset, genomeId, generateHtoCalls ? CellHashingHandler.BARCODE_TYPE.hashing : CellHashingHandler.BARCODE_TYPE.citeseq, createOutputFiles);`
	`535`	`+ return handler.runCiteSeqCount(output, outputCategory, htoReadset, htoList, cellBarcodeList, outputDir, basename, log, extraArgs, doHtoFiltering, minCountPerCell, localPipelineDir, editDistance, scanEditDistances, parentReadset, genomeId, generateHtoCalls ? CellHashingHandler.BARCODE_TYPE.hashing : CellHashingHandler.BARCODE_TYPE.citeseq, createOutputFiles, useSeurat, useMultiSeq);`
`536`	`536`	`}`
`537`	`537`	`}`
Original file line number	Diff line number	Diff line change
`@@ -444,7 +444,7 @@ else if (line.startsWith("Percentage unmapped"))`
`444`	`444`	`if (type.doGenerateCalls())`
`445`	`445`	`{`
`446`	`446`	`ctx.getJob().setStatus(PipelineJob.TaskStatus.running, "Generating HTO calls for edit distance: " + editDistance);`
`447`		`- File htoCalls = generateFinalCalls(outputMatrix.getParentFile(), ctx.getOutputDir(), outputBasename, ctx.getLogger(), null, true, minCountPerCell, ctx.getSourceDirectory());`
	`447`	`+ File htoCalls = generateFinalCalls(outputMatrix.getParentFile(), ctx.getOutputDir(), outputBasename, ctx.getLogger(), null, true, minCountPerCell, ctx.getSourceDirectory(), true, true);`
`448`	`448`	`File html = new File(htoCalls.getParentFile(), outputBasename + ".html");`
`449`	`449`
`450`	`450`	`if (!html.exists())`
`@@ -665,7 +665,7 @@ private File ensureLocalCopy(File input, File outputDir, Logger log, Set<File> t`
`665`	`665`	`return input;`
`666`	`666`	`}`
`667`	`667`
`668`		`- public File generateFinalCalls(File citeSeqCountOutDir, File outputDir, String basename, Logger log, @Nullable File cellBarcodeWhitelist, boolean doHtoFiltering, Integer minCountPerCell, File localPipelineDir) throws PipelineJobException`
	`668`	`+ private File generateFinalCalls(File citeSeqCountOutDir, File outputDir, String basename, Logger log, @Nullable File cellBarcodeWhitelist, boolean doHtoFiltering, Integer minCountPerCell, File localPipelineDir, boolean useSeurat, boolean useMultiSeq) throws PipelineJobException`
`669`	`669`	`{`
`670`	`670`	`log.debug("generating final calls from folder: " + citeSeqCountOutDir.getPath());`
`671`	`671`
`@@ -695,6 +695,9 @@ public File generateFinalCalls(File citeSeqCountOutDir, File outputDir, String b`
`695`	`695`	`File rawCallsFile = new File(outputDir, basename + ".raw.txt");`
`696`	`696`	`File metricsFile = getMetricsFile(callsFile);`
`697`	`697`	`List<String> args = new ArrayList<>(Arrays.asList("/bin/bash", scriptWrapper, citeSeqCountOutDir.getName(), htmlFile.getName(), callsFile.getName(), rawCallsFile.getName(), (doHtoFiltering ? "T" : "F"), (minCountPerCell == null ? "0" : minCountPerCell.toString()), metricsFile.getName()));`
	`698`	`+ args.add(useSeurat ? "TRUE" : "FALSE");`
	`699`	`+ args.add(useMultiSeq ? "TRUE" : "FALSE");`
	`700`	`+`
`698`	`701`	`if (cellBarcodeWhitelist != null)`
`699`	`702`	`{`
`700`	`703`	`args.add(cellBarcodeWhitelist.getName());`
`@@ -877,7 +880,7 @@ else if (htoReadset.getReadData().size() != 1)`
`877`	`880`	`return ret;`
`878`	`881`	`}`
`879`	`882`
`880`		`- public File runCiteSeqCount(PipelineStepOutput output, String category, Readset htoReadset, File htoList, File cellBarcodeList, File outputDir, String basename, Logger log, List<String> extraArgs, boolean doHtoFiltering, @Nullable Integer minCountPerCell, File localPipelineDir, @Nullable Integer editDistance, boolean scanEditDistances, Readset parentReadset, @Nullable Integer genomeId, BARCODE_TYPE type, boolean createOutputFiles) throws PipelineJobException`
	`883`	+ public File runCiteSeqCount(PipelineStepOutput output, String category, Readset htoReadset, File htoList, File cellBarcodeList, File outputDir, String basename, Logger log, List<String> extraArgs, boolean doHtoFiltering, @Nullable Integer minCountPerCell, File localPipelineDir, @Nullable Integer editDistance, boolean scanEditDistances, Readset parentReadset, @Nullable Integer genomeId, BARCODE_TYPE type, boolean createOutputFiles, boolean useSeurat, boolean useMultiSeq) throws PipelineJobException
`881`	`884`	`{`
`882`	`885`	`HtoMergeResult htoFastqs = possiblyMergeHtoFastqs(htoReadset, outputDir, log);`
`883`	`886`	`if (!htoFastqs.intermediateFiles.isEmpty())`
`@@ -964,7 +967,7 @@ public File runCiteSeqCount(PipelineStepOutput output, String category, Readset`
`964`	`967`
`965`	`968`	`File citeSeqCountOutDir = new File(outputDir, basename + ".citeSeqCounts." + ed + "." + type.name());`
`966`	`969`	`String outputBasename = basename + "." + ed + "." + type.name();`
`967`		`- Map<String, Object> callMap = executeCiteSeqCountWithJobCtx(outputDir, outputBasename, citeSeqCountOutDir, htoFastqs.files.getLeft(), htoFastqs.files.getRight(), toolArgs, ed, log, cellBarcodeList, doHtoFiltering, minCountPerCell, localPipelineDir, unknownBarcodeFile, type);`
	`970`	`+ Map<String, Object> callMap = executeCiteSeqCountWithJobCtx(outputDir, outputBasename, citeSeqCountOutDir, htoFastqs.files.getLeft(), htoFastqs.files.getRight(), toolArgs, ed, log, cellBarcodeList, doHtoFiltering, minCountPerCell, localPipelineDir, unknownBarcodeFile, type, useSeurat, useMultiSeq);`
`968`	`971`	`results.put(ed, callMap);`
`969`	`972`
`970`	`973`	`if (type.doGenerateCalls())`
`@@ -1065,7 +1068,7 @@ public File runCiteSeqCount(PipelineStepOutput output, String category, Readset`
`1065`	`1068`	`}`
`1066`	`1069`	`}`
`1067`	`1070`
`1068`		`- private Map<String, Object> executeCiteSeqCountWithJobCtx(File outputDir, String basename, File citeSeqCountOutDir, File fastq1, File fastq2, List<String> baseArgs, Integer ed, Logger log, File cellBarcodeList, boolean doHtoFiltering, Integer minCountPerCell, File localPipelineDir, File unknownBarcodeFile, BARCODE_TYPE type) throws PipelineJobException`
	`1071`	`+ private Map<String, Object> executeCiteSeqCountWithJobCtx(File outputDir, String basename, File citeSeqCountOutDir, File fastq1, File fastq2, List<String> baseArgs, Integer ed, Logger log, File cellBarcodeList, boolean doHtoFiltering, Integer minCountPerCell, File localPipelineDir, File unknownBarcodeFile, BARCODE_TYPE type, boolean useSeurat, boolean useMultiSeq) throws PipelineJobException`
`1069`	`1072`	`{`
`1070`	`1073`	`CellHashingHandler.CiteSeqCountWrapper wrapper = new CellHashingHandler.CiteSeqCountWrapper(log);`
`1071`	`1074`	`File doneFile = new File(citeSeqCountOutDir, "citeSeqCount." + type.name() + "." + ed + ".done");`
`@@ -1132,7 +1135,7 @@ else if (line.startsWith("Percentage unmapped"))`
`1132`	`1135`
`1133`	`1136`	`if (type.doGenerateCalls())`
`1134`	`1137`	`{`
`1135`		`- File htoCalls = generateFinalCalls(outputMatrix.getParentFile(), outputDir, basename, log, cellBarcodeList, doHtoFiltering, minCountPerCell, localPipelineDir);`
	`1138`	`+ File htoCalls = generateFinalCalls(outputMatrix.getParentFile(), outputDir, basename, log, cellBarcodeList, doHtoFiltering, minCountPerCell, localPipelineDir, useSeurat, useMultiSeq);`
`1136`	`1139`	`if (!htoCalls.exists())`
`1137`	`1140`	`{`
`1138`	`1141`	`throw new PipelineJobException("missing expected file: " + htoCalls.getPath());`
Original file line number	Diff line number	Diff line change
`@@ -143,7 +143,7 @@ public RecordedActionSet run() throws PipelineJobException`
`143`	`143`	`}`
`144`	`144`
`145`	`145`	`final int trackId = addTrackForLibrary(getPipelineJob().getTrack(), getPipelineJob().getTrackName(), getPipelineJob().getTrackDescription(), action);`
`146`		`- ReferenceGenomeManager.get().markGenomeModified(SequenceAnalysisService.get().getReferenceGenome(getPipelineJob().getLibraryId(), getJob().getUser()));`
	`146`	`+ ReferenceGenomeManager.get().markGenomeModified(SequenceAnalysisService.get().getReferenceGenome(getPipelineJob().getLibraryId(), getJob().getUser()), getJob().getLogger());`
`147`	`147`
`148`	`148`	`Set<GenomeTrigger> triggers = SequenceAnalysisServiceImpl.get().getGenomeTriggers();`
`149`	`149`	`if (!triggers.isEmpty())`