Allow pipeline to preferentially use previously generated cellbender filtered matrix

Author: bbimber

singlecell/resources/chunks/PrepareRawCounts.R

@@ -2,7 +2,18 @@ for (datasetId in names(seuratObjects)) {
     rawCountDir <- seuratObjects[[datasetId]]
 
     datasetName <- datasetIdToName[[datasetId]]
-    seuratObj <- CellMembrane::ReadAndFilter10xData(dataDir = rawCountDir, datasetId = datasetId, datasetName = datasetName, emptyDropsLower = emptyDropsLower, emptyDropsFdrThreshold = emptyDropsFdrThreshold, useEmptyDropsCellRanger = useEmptyDropsCellRanger, nExpectedCells = nExpectedCells, useCellBender = useCellBender, useSoupX = useSoupX)
+
+    previouslyFilteredMatrix <- NULL
+    if (useCellBender) {
+        # This is the 10x project:
+        print('Will use cellbender-adjusted counts instead of raw counts:')
+        previouslyFilteredMatrix <- paste0(rawCountDir, 'raw_feature_bc_matrix.cellbender_filtered.h5')
+        if (!file.exists(previouslyFilteredMatrix)) {
+            stop(paste0('Unable to find file: ', previouslyFilteredMatrix, '. You can re-run cellbender to fix this.'))
+        }
+    }
+
+    seuratObj <- CellMembrane::ReadAndFilter10xData(dataDir = rawCountDir, datasetId = datasetId, datasetName = datasetName, emptyDropsLower = emptyDropsLower, emptyDropsFdrThreshold = emptyDropsFdrThreshold, useEmptyDropsCellRanger = useEmptyDropsCellRanger, nExpectedCells = nExpectedCells, useSoupX = useSoupX, previouslyFilteredMatrix = previouslyFilteredMatrix)
 
     if (!is.null(maxAllowableCells) && ncol(seuratObj) > maxAllowableCells) {
         addErrorMessage(paste0('The seurat object has ', ncol(seuratObj), ' cells, which is more than the max allowable cells (', maxAllowableCells, '). Please review emptyDrops results as this probably means thresholds were suboptimal.'))

singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java

@@ -36,6 +36,7 @@
 import org.labkey.api.singlecell.CellHashingService;
 import org.labkey.api.singlecell.pipeline.AbstractSingleCellPipelineStep;
 import org.labkey.api.singlecell.pipeline.AbstractSingleCellStep;
+import org.labkey.api.singlecell.pipeline.SingleCellRawDataStep;
 import org.labkey.api.singlecell.pipeline.SingleCellStep;
 import org.labkey.api.util.FileUtil;
 import org.labkey.api.util.PageFlowUtil;
@@ -144,6 +145,16 @@ public Processor(boolean doProcessRawCounts)
         @Override
         public void init(JobContext ctx, List<SequenceOutputFile> inputFiles, List<RecordedAction> actions, List<SequenceOutputFile> outputsToCreate) throws UnsupportedOperationException, PipelineJobException
         {
+            List<PipelineStepCtx<SingleCellRawDataStep>> rawCountSteps = SequencePipelineService.get().getSteps(ctx.getJob(), SingleCellRawDataStep.class);
+            if (!rawCountSteps.isEmpty())
+            {
+                for (PipelineStepCtx<SingleCellRawDataStep> stepCtx : rawCountSteps)
+                {
+                    SingleCellRawDataStep step = stepCtx.getProvider().create(ctx);
+                    step.init(ctx, inputFiles);
+                }
+            }
+
             boolean requiresHashing = false;
             boolean requiresCite = false;
             boolean doH5Caching = false;

singlecell/src/org/labkey/singlecell/pipeline/singlecell/PrepareRawCounts.java

@@ -1,20 +1,20 @@
 package org.labkey.singlecell.pipeline.singlecell;
 
-import org.apache.logging.log4j.Logger;
 import org.json.JSONObject;
 import org.junit.Assert;
 import org.junit.Test;
-import org.labkey.api.pipeline.PipelineJob;
-import org.labkey.api.pipeline.WorkDirectory;
+import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.sequenceanalysis.SequenceOutputFile;
 import org.labkey.api.sequenceanalysis.pipeline.AbstractPipelineStepProvider;
 import org.labkey.api.sequenceanalysis.pipeline.PipelineContext;
-import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport;
+import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler;
 import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
 import org.labkey.api.singlecell.pipeline.SeuratToolParameter;
 import org.labkey.api.singlecell.pipeline.SingleCellRawDataStep;
 
 import java.io.File;
 import java.util.Arrays;
+import java.util.List;
 
 public class PrepareRawCounts extends AbstractCellMembraneStep
 {
@@ -60,6 +60,25 @@ public PrepareRawCounts create(PipelineContext ctx)
         }
     }
 
+    @Override
+    public void init(SequenceOutputHandler.JobContext ctx, List<SequenceOutputFile> inputFiles) throws PipelineJobException
+    {
+        for (SequenceOutputFile so : inputFiles)
+        {
+            boolean useCellBender = getProvider().getParameterByName("useCellBender").extractValue(ctx.getJob(), getProvider(), getStepIdx(), Boolean.class, false);
+            if (useCellBender)
+            {
+                // This is the outs dir:
+                File source = so.getFile().getParentFile();
+                File expected = new File(source, "raw_feature_bc_matrix.cellbender_filtered.h5");
+                if (!expected.exists())
+                {
+                    throw new IllegalArgumentException("Missing cellbender-corrected matrix. You can re-run cellbender on the loupe file to fix this: " + expected.getPath());
+                }
+            }
+        }
+    }
+
     @Override
     public String getFileSuffix()
     {