Support demuxEM demultiplexing, including caching of H5 files

Author: bbimber

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java

@@ -471,7 +471,9 @@ public static File processVCF(File input, Integer libraryId, JobContext ctx, Res
 
     private static String countUsingBcfTools(File vcf, Logger log) throws PipelineJobException
     {
-        return new BcftoolsRunner(log).executeWithOutput(Arrays.asList("index", "-n", vcf.getPath()));
+        BcftoolsRunner wrapper = new BcftoolsRunner(log);
+
+        return wrapper.executeWithOutput(Arrays.asList(wrapper.getBcfToolsPath().getPath(), "index", "-n", vcf.getPath()));
     }
 
     public static String getVCFLineCount(File vcf, Logger log, boolean passOnly, boolean useBcfTools) throws PipelineJobException

singlecell/api-src/org/labkey/api/singlecell/CellHashingService.java

@@ -2,7 +2,6 @@
 
 import au.com.bytecode.opencsv.CSVReader;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.json.JSONObject;
@@ -45,12 +44,12 @@ static public void setInstance(CellHashingService instance)
         _instance = instance;
     }
 
-    abstract public void prepareHashingIfNeeded(File sourceDir, PipelineJob job, SequenceAnalysisJobSupport support, String filterField, final boolean failIfNoHashing) throws PipelineJobException;
-
-    abstract public void prepareHashingAndCiteSeqFilesIfNeeded(File sourceDir, PipelineJob job, SequenceAnalysisJobSupport support, String filterField, boolean failIfNoHashing, boolean failIfNoCiteSeq) throws PipelineJobException;
+    abstract public void prepareHashingForVdjIfNeeded(File sourceDir, PipelineJob job, SequenceAnalysisJobSupport support, String filterField, final boolean failIfNoHashing) throws PipelineJobException;
 
     abstract public File generateHashingCallsForRawMatrix(Readset parentReadset, PipelineOutputTracker output, SequenceOutputHandler.JobContext ctx, CellHashingParameters parameters, File rawCountMatrixDir) throws PipelineJobException;
 
+    abstract public File getH5FileForGexReadset(SequenceAnalysisJobSupport support, int readsetId, int genomeId) throws PipelineJobException;
+
     abstract public File getCDNAInfoFile(File sourceDir);
 
     abstract public Sample getSampleById(int rowId);
@@ -69,7 +68,7 @@ static public void setInstance(CellHashingService instance)
 
     abstract public boolean usesCiteSeq(SequenceAnalysisJobSupport support, List<SequenceOutputFile> inputFiles) throws PipelineJobException;
 
-    abstract public List<ToolParameterDescriptor> getHashingCallingParams();
+    abstract public List<ToolParameterDescriptor> getHashingCallingParams(boolean allowDemuxEm);
 
     abstract public Set<String> getHtosForParentReadset(Integer parentReadsetId, File webserverJobDir, SequenceAnalysisJobSupport support) throws PipelineJobException;
 
@@ -94,10 +93,12 @@ public static class CellHashingParameters
         public @Nullable Integer genomeId;
         public boolean skipNormalizationQc = false;
         public Integer minCountPerCell = 5;
-        public List<CALLING_METHOD> methods = CALLING_METHOD.getDefaultMethods();
+        public List<CALLING_METHOD> methods = CALLING_METHOD.getDefaultConsensusMethods(); //Default to just executing the set used for default consensus calls, rather than additional ones
+        public List<CALLING_METHOD> consensusMethods = null;
         public String basename = null;
         public Integer cells = 0;
         public boolean keepMarkdown = false;
+        public File h5File = null;
 
         private CellHashingParameters()
         {
@@ -127,6 +128,14 @@ public static CellHashingService.CellHashingParameters createFromStep(SequenceOu
                 {
                     throw new IllegalArgumentException("Must provide at least one calling method");
                 }
+
+                String methodStr2 = StringUtils.trimToNull(step.getProvider().getParameterByName("consensusMethods").extractValue(ctx.getJob(), step.getProvider(), step.getStepIdx(), String.class));
+                if (methodStr2 != null)
+                {
+                    ret.consensusMethods = extractMethods(methodStr2);
+
+                    throw new PipelineJobException("All consensusMethods must be present in methods: " + methodStr2);
+                }
             }
 
             return ret;
@@ -143,12 +152,21 @@ public static CellHashingParameters createFromJson(BARCODE_TYPE type, File webse
             ret.parentReadset = parentReadset;
             ret.htoBarcodesFile = new File(webserverDir, type.getAllBarcodeFileName());
             ret.methods = extractMethods(params.optString("methods"));
+            ret.consensusMethods = extractMethods(params.optString("consensusMethods"));
 
             if (type == BARCODE_TYPE.hashing && ret.methods.isEmpty())
             {
                 throw new IllegalArgumentException("Must provide at least one calling method");
             }
 
+            if (ret.consensusMethods != null && !ret.consensusMethods.isEmpty())
+            {
+                if (!ret.methods.containsAll(ret.consensusMethods))
+                {
+                    throw new PipelineJobException("All consensusMethods must be present in methods: " + ret.consensusMethods.stream().map(CALLING_METHOD::name).collect(Collectors.joining(",")));
+                }
+            }
+
             return ret;
         }
 
@@ -278,33 +296,51 @@ public Set<String> getAllowableBarcodeNames() throws PipelineJobException
 
     public enum CALLING_METHOD
     {
-        multiseq(false),
-        htodemux(false),
-        dropletutils(true),
-        gmm_demux(true),
-        bff_cluster(true),
-        bff_raw(false);
+        multiseq(true, false),
+        htodemux(false, false),
+        dropletutils(true, true),
+        gmm_demux(true, true),
+        demuxem(true, false),
+        bff_cluster(true, true),
+        bff_raw(true, false);
+
+        boolean isDefaultRun;
+        boolean isDefaultConsensus;
+
+        CALLING_METHOD(boolean isDefaultRun, boolean isDefaultConsensus)
+        {
+            this.isDefaultRun = isDefaultRun;
+            this.isDefaultConsensus = isDefaultConsensus;
+        }
 
-        boolean isDefault;
+        public boolean isDefaultRun()
+        {
+            return isDefaultRun;
+        }
+
+        public boolean isDefaultConsensus()
+        {
+            return isDefaultConsensus;
+        }
 
-        CALLING_METHOD(boolean isDefault)
+        private static List<CALLING_METHOD> getDefaultRunMethods()
         {
-            this.isDefault = isDefault;
+            return Arrays.stream(values()).filter(CALLING_METHOD::isDefaultRun).collect(Collectors.toList());
         }
 
-        public boolean isDefault()
+        private static List<CALLING_METHOD> getDefaultConsensusMethods()
         {
-            return isDefault;
+            return Arrays.stream(values()).filter(CALLING_METHOD::isDefaultConsensus).collect(Collectors.toList());
         }
 
-        public static List<CALLING_METHOD> getDefaultMethods()
+        public static List<String> getDefaultRunMethodNames()
         {
-            return Arrays.stream(values()).filter(CALLING_METHOD::isDefault).collect(Collectors.toList());
+            return getDefaultRunMethods().stream().map(Enum::name).collect(Collectors.toList());
         }
 
-        public static List<String> getDefaultMethodNames()
+        public static List<String> getDefaultConsensusMethodNames()
         {
-            return getDefaultMethods().stream().map(Enum::name).collect(Collectors.toList());
+            return getDefaultConsensusMethods().stream().map(Enum::name).collect(Collectors.toList());
         }
     }