Merge discvr19.1 r64416-64455

SVN r64456 |2019-09-08 14:35:56 +0000

Author: bbimber

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/SequenceAnalysisService.java

@@ -90,4 +90,6 @@ static public void setInstance(SequenceAnalysisService instance)
     abstract public File getPicardJar();
 
     abstract public File writeAllCellHashingBarcodes(File webserverDir) throws PipelineJobException;
+
+    abstract public String createReferenceLibrary(List<Integer> sequenceIds, Container c, User u, String name, String assemblyId, String description, boolean skipCacheIndexes, boolean skipTriggers) throws IOException;
 }

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/VariantProcessingStep.java

@@ -15,9 +15,12 @@
  */
 package org.labkey.api.sequenceanalysis.pipeline;
 
+import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.sequenceanalysis.SequenceOutputFile;
 
 import java.io.File;
+import java.util.List;
 
 /**
  * User: bimber
@@ -28,6 +31,11 @@ public interface VariantProcessingStep extends PipelineStep
 {
     public Output processVariants(File inputVCF, File outputDirectory, ReferenceGenome genome) throws PipelineJobException;
 
+    default void init(PipelineJob job, SequenceAnalysisJobSupport support, List<SequenceOutputFile> inputFiles) throws PipelineJobException
+    {
+
+    }
+
     public static interface Output extends PipelineStepOutput
     {
         public File getVCF();

SequenceAnalysis/resources/external/scRNAseq/Seurat3.rmd

@@ -1,4 +1,4 @@
-```{r}
+```{r Setup}
 
 knitr::opts_chunk$set(message=FALSE, warning=FALSE,echo=TRUE,error = FALSE)
 library(knitr)
@@ -15,6 +15,15 @@ if (cores != ''){
 print('Updating future.globals.maxSize')
 options(future.globals.maxSize = Inf)
 
+print('Global variables: ')
+for (v in c('outPrefix', 'resolutionToUse', 'dimsToUse', 'minDimsToUse', 'doCellFilter', 'doCellCycle', 'useSCTransform', 'mergeMethod')){
+    if (exists(v)){
+        print(paste0(v, ': ', get(v)))
+    } else {
+      print(paste0(v, ': not defined'))
+    }
+}
+
 ```
 
 ## Prepare data
@@ -49,7 +58,7 @@ if (file.exists(saveFile)) {
   print('resuming from file')
   seuratObj <- readRDS(saveFile)
 } else {
-  seuratObj <- MergeSeuratObjs(seuratObjs, metadata = data)
+  seuratObj <- MergeSeuratObjs(seuratObjs, metadata = data, method = mergeMethod)
   saveRDS(seuratObj, file = saveFile)
   rm(seuratObjs)
 }
@@ -70,15 +79,16 @@ seuratObj <- ProcessSeurat1(seuratObj, variableGeneTable = paste0(outPrefix, '.v
 
 ```{r DimRedux}
 
-seuratObj <- FindClustersAndDimRedux(seuratObj, dimsToUse = dimsToUse)
+seuratObj <- FindClustersAndDimRedux(seuratObj, dimsToUse = dimsToUse, minDimsToUse = minDimsToUse)
+saveRDS(seuratObj, file = saveFile)
 
 Find_Markers(seuratObj, resolutionToUse = resolutionToUse, outFile = paste0(outPrefix, '.markers.txt'), saveFileMarkers = paste0(outPrefix, '.markers.rds'))
 
 ```
 
 ## SingleR
 
-```{r}
+```{r SingleR}
 
 seuratObj <- RunSingleR(seuratObj = seuratObj, resultTableFile = paste0(outPrefix, '.singleR.txt'))
 saveRDS(seuratObj, file = saveFile)
@@ -91,17 +101,27 @@ Tabulate_SingleRClassLabs(seuratObj, plotIndividually = T)
 
 ## Phenotypes
 
-```{r}
+```{r Phenotypes}
 
 PlotImmuneMarkers(seuratObj, reduction = 'tsne')
 
 PlotImmuneMarkers(seuratObj, reduction = 'umap')
 
+if (length(unique(seuratObj$BarcodePrefix)) > 1) {
+    print(DimPlot(seuratObj, reduction = 'pca', group.by = 'BarcodePrefix', label = T))
+    print(DimPlot(seuratObj, reduction = 'tsne', group.by = 'BarcodePrefix', label = T))
+    print(DimPlot(seuratObj, reduction = 'umap', group.by = 'BarcodePrefix', label = T))
+
+    t <- table(Cluster = Idents(seuratObj), Dataset = seuratObj$BarcodePrefix)
+    t <- round(t / colSums(t), 2)
+    knitr::kable(t)
+}
+
 ```
 
 ## Write Summary
 
-```{r}
+```{r Summary}
 
 saveRDS(seuratObj, file = saveFile)
 unlink(rawDataSaveFile)

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java

@@ -4379,6 +4379,12 @@ public ApiResponse execute(CompareFastaSequencesForm form, BindException errors)
                             continue;
                         }
 
+                        if (m.getSeqLength() > 1000000)
+                        {
+                            _log.info("skipping large reference: " + m.getName());
+                            continue;
+                        }
+
                         SequenceMatch sm = SequenceMatch.checkReference(fastaHeader, fastaData.get(fastaHeader), m);
                         if (sm != null)
                         {
@@ -4396,6 +4402,12 @@ public ApiResponse execute(CompareFastaSequencesForm form, BindException errors)
                                 continue;
                             }
 
+                            if (m.getSeqLength() > 1000000)
+                            {
+                                _log.info("skipping large reference: " + m.getName());
+                                continue;
+                            }
+
                             SequenceMatch sm = SequenceMatch.checkReference(fastaHeader, fastaData.get(fastaHeader), m);
                             if (sm != null)
                             {

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisServiceImpl.java

@@ -40,6 +40,7 @@
 import org.labkey.sequenceanalysis.analysis.CellHashingHandler;
 import org.labkey.sequenceanalysis.pipeline.ProcessVariantsHandler;
 import org.labkey.sequenceanalysis.pipeline.ReferenceGenomeImpl;
+import org.labkey.sequenceanalysis.pipeline.ReferenceLibraryPipelineJob;
 import org.labkey.sequenceanalysis.pipeline.SequenceTaskHelper;
 import org.labkey.sequenceanalysis.run.util.BgzipRunner;
 import org.labkey.sequenceanalysis.run.util.FastaIndexer;
@@ -453,4 +454,12 @@ public File writeAllCellHashingBarcodes(File webserverDir) throws PipelineJobExc
     {
         return CellHashingHandler.writeAllBarcodes(webserverDir);
     }
+
+    @Override
+    public String createReferenceLibrary(List<Integer> sequenceIds, Container c, User u, String name, String assemblyId, String description, boolean skipCacheIndexes, boolean skipTriggers) throws IOException
+    {
+        ReferenceLibraryPipelineJob job = SequenceAnalysisManager.get().createReferenceLibrary(sequenceIds, c, u, name, assemblyId, description, skipCacheIndexes, skipTriggers, null);
+
+        return job.getJobGUID();
+    }
 }

SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/CellRangerSeuratHandler.java

@@ -67,8 +67,11 @@ public CellRangerSeuratHandler()
                 }}, true),
                 ToolParameterDescriptor.create("useSCTransform", "Use SCTransform", "If selected, the pipeline will use the newer SCtransform method instead of the standard Seurat pipeline.", "checkbox", new JSONObject(){{
                     put("checked", false);
-                }}, false)
-        ));
+                }}, false),
+                ToolParameterDescriptor.create("mergeMethod", "Merge Method", "This determines whether any batch correction will be applied when merging datasets.", "ldk-simplecombo", new JSONObject(){{
+                    put("storeValues", "simple;cca");
+                }}, "simple")
+            ));
     }
 
     @Override
@@ -245,8 +248,6 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
             outPrefix = FileUtil.makeLegalName(outPrefix);
 
             File tmpScript = new File(ctx.getWorkingDirectory(), "script.R");
-            ctx.getFileManager().addIntermediateFile(tmpScript);
-
             File outHtml = new File(ctx.getWorkingDirectory(), outPrefix + ".html");
 
             try (PrintWriter writer = PrintWriters.getPrintWriter(tmpScript))
@@ -272,10 +273,13 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
 
                 writer.println("outPrefix <- '" + outPrefix + "'");
                 writer.println("resolutionToUse <- 0.6");
-                String dimsToUse = StringUtils.trimToNull(ctx.getParams().optString("dimsToUse"));
-                dimsToUse = dimsToUse == null ? "NULL" : dimsToUse;
+                for (String v : new String[]{"dimsToUse", "minDimsToUse", "mergeMethod"}) {
+                    String val = StringUtils.trimToNull(ctx.getParams().optString(v));
+                    val = val == null ? "NULL" : val;
+
+                    writer.println(v + " <- " + val);
+                }
 
-                writer.println("dimsToUse <- " + dimsToUse);
                 boolean doCellFilter = ctx.getParams().optBoolean("doCellFilter", true);
                 writer.println("doCellFilter <- " + String.valueOf(doCellFilter).toUpperCase());
 
@@ -315,11 +319,16 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
             }
 
             String dimsToUse = StringUtils.trimToNull(ctx.getParams().optString("dimsToUse"));
+            String minDimsToUse = StringUtils.trimToNull(ctx.getParams().optString("minDimsToUse"));
+            String mergeMethod = StringUtils.trimToNull(ctx.getParams().optString("mergeMethod"));
+
             String description = StringUtils.join(new String[]{
                     "Correct Cell Cycle: " + ctx.getParams().optBoolean("doCellCycle", true),
                     "Perform Cell Filtering: " + ctx.getParams().optBoolean("doCellFilter", true),
+                    "Min. Dims To Use: " + (minDimsToUse == null ? "NA" : minDimsToUse),
                     "Dims To Use: " + (dimsToUse == null ? "automatic" : dimsToUse),
-                    "Use SCTransform: " + ctx.getParams().optBoolean("useSCTransform", false)
+                    "Use SCTransform: " + ctx.getParams().optBoolean("useSCTransform", false),
+                    "Merge method: " + mergeMethod
             }, "\n");
 
             ctx.getFileManager().addSequenceOutput(seuratObj, "Seurat Object: " + outPrefix, "Seurat Data", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), description);
@@ -330,7 +339,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
                 throw new PipelineJobException("Unable to find summary report");
             }
             ctx.getFileManager().addOutput(action, "Seurat Report", outHtml);
-            ctx.getFileManager().addSequenceOutput(outHtml, "Seurat Report: " + outPrefix, "Seurat Report", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), null);
+            ctx.getFileManager().addSequenceOutput(outHtml, "Seurat Report: " + outPrefix, "Seurat Report", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), description);
 
             File seuratObjRaw = new File(ctx.getWorkingDirectory(), outPrefix + ".rawData.rds");
             if (seuratObjRaw.exists())

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java

@@ -138,8 +138,15 @@ public SequenceOutputProcessor getProcessor()
         return new Processor();
     }
 
+    private static SequenceOutputHandlerJob getPipelineJob(PipelineJob job)
+    {
+        return (SequenceOutputHandlerJob)job;
+    }
+
     public static void initVariantProcessing(PipelineJob job, SequenceAnalysisJobSupport support, List<SequenceOutputFile> inputFiles, File outputDir) throws PipelineJobException
     {
+        SequenceTaskHelper taskHelper = new SequenceTaskHelper(getPipelineJob(job), outputDir);
+
         List<PipelineStepCtx<VariantProcessingStep>> providers = SequencePipelineService.get().getSteps(job, VariantProcessingStep.class);
         boolean requiresPedigree = false;
         for (PipelineStepCtx<VariantProcessingStep> stepCtx : providers)
@@ -158,6 +165,9 @@ public static void initVariantProcessing(PipelineJob job, SequenceAnalysisJobSup
             {
                 requiresPedigree = true;
             }
+
+            VariantProcessingStep step = stepCtx.getProvider().create(taskHelper);
+            step.init(job, support, inputFiles);
         }
 
         if (requiresPedigree)