Add check for hashing and HTOsDroppedAboveMinRetained

Author: bbimber

singlecell/api-src/org/labkey/api/singlecell/CellHashingService.java

@@ -64,6 +64,8 @@ static public void setInstance(CellHashingService instance)
 
     abstract public void processMetrics(SequenceOutputFile so, PipelineJob job, boolean updateDescription) throws PipelineJobException;
 
+    abstract public File getMetricsFile(File callFile);
+
     /**
      * Using the sample metadata, return whether cell hashing is used.
      * NOTE: if readset ID is null, this will be interpreted as whether any readset in the input data uses hashing
@@ -92,6 +94,7 @@ public static class CellHashingParameters
         public boolean createOutputFiles = true;
         public @Nullable String outputCategory;
         public boolean retainRawCountFile = false;
+        public boolean failIfUnexpectedHtosFound = true;
 
         public Readset htoReadset;
         public Readset parentReadset;
@@ -124,6 +127,7 @@ public static CellHashingService.CellHashingParameters createFromStep(SequenceOu
             ret.callerDisagreementThreshold = step.getProvider().getParameterByName("callerDisagreementThreshold").extractValue(ctx.getJob(), step.getProvider(), step.getStepIdx(), Double.class, null);
             ret.doTSNE = step.getProvider().getParameterByName("doTSNE").extractValue(ctx.getJob(), step.getProvider(), step.getStepIdx(), Boolean.class, null);
             ret.retainRawCountFile = step.getProvider().getParameterByName("retainRawCountFile").extractValue(ctx.getJob(), step.getProvider(), step.getStepIdx(), Boolean.class, true);
+            ret.failIfUnexpectedHtosFound = step.getProvider().getParameterByName("failIfUnexpectedHtosFound").extractValue(ctx.getJob(), step.getProvider(), step.getStepIdx(), Boolean.class, true);
             ret.htoReadset = htoReadset;
             ret.parentReadset = parentReadset;
             ret.htoBarcodesFile = new File(ctx.getSourceDirectory(), type.getAllBarcodeFileName());
@@ -165,6 +169,7 @@ public static CellHashingParameters createFromJson(BARCODE_TYPE type, File webse
             ret.callerDisagreementThreshold = params.get("callerDisagreementThreshold") == null ? null : params.getDouble("callerDisagreementThreshold");
             ret.doTSNE = params.get("doTSNE") == null || params.getBoolean("doTSNE");
             ret.retainRawCountFile = params.optBoolean("retainRawCountFile", true);
+            ret.failIfUnexpectedHtosFound = params.optBoolean("failIfUnexpectedHtosFound", true);
             ret.htoReadset = htoReadset;
             ret.parentReadset = parentReadset;
             ret.htoBarcodesFile = new File(webserverDir, type.getAllBarcodeFileName());
@@ -280,6 +285,7 @@ public Set<String> getAllowableBarcodeNames() throws PipelineJobException
             {
                 return Collections.unmodifiableSet(allowableHtoBarcodes);
             }
+
             if (htoBarcodesFile == null)
             {
                 throw new IllegalArgumentException("Barcode file was null");

singlecell/src/org/labkey/singlecell/CellHashingServiceImpl.java

@@ -804,6 +804,7 @@ public Sort getSortById(int rowId)
         return new TableSelector(SingleCellSchema.getInstance().getSchema().getTable(SingleCellSchema.TABLE_SORTS)).getObject(rowId, Sort.class);
     }
 
+    @Override
     public File getMetricsFile(File callFile)
     {
         return new File(callFile.getPath().replaceAll(CALL_EXTENSION, ".metrics.txt"));
@@ -951,7 +952,8 @@ public List<ToolParameterDescriptor> getHashingCallingParams(boolean allowMethod
             }}, 0.2),
             ToolParameterDescriptor.create("skipNormalizationQc", "Skip Normalization QC", null, "checkbox", null, true),
             ToolParameterDescriptor.create("doTSNE", "Do tSNE", "If true, tSNE will be performed as part of QC", "checkbox", null, true),
-            ToolParameterDescriptor.create("retainRawCountFile", "Retain Raw Counts File", null, "checkbox", null, false)
+            ToolParameterDescriptor.create("retainRawCountFile", "Retain Raw Counts File", null, "checkbox", null, false),
+            ToolParameterDescriptor.create("failIfUnexpectedHtosFound", "Fail If Unexpected HTOs Found", "If checked and if there are any HTOs (testing all known HTOs) with counts above the HTOs expected in this experiment, then an error will be thrown", "checkbox", null, false)
         ));
 
         final List<String> allMethods = Arrays.stream(CALLING_METHOD.values()).filter(x -> allowMethodsNeedingGex || !x.isRequiresH5()).map(Enum::name).toList();

singlecell/src/org/labkey/singlecell/pipeline/singlecell/AbstractCellHashingCiteseqStep.java

@@ -19,7 +19,7 @@
 
 abstract public class AbstractCellHashingCiteseqStep extends AbstractSingleCellPipelineStep
 {
-    public AbstractCellHashingCiteseqStep(PipelineStepProvider provider, PipelineContext ctx)
+    public AbstractCellHashingCiteseqStep(PipelineStepProvider<?> provider, PipelineContext ctx)
     {
         super(provider, ctx);
     }

singlecell/src/org/labkey/singlecell/pipeline/singlecell/RunCellHashing.java

@@ -1,5 +1,6 @@
 package org.labkey.singlecell.pipeline.singlecell;
 
+import au.com.bytecode.opencsv.CSVReader;
 import org.apache.commons.io.FileUtils;
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.reader.Readers;
@@ -134,6 +135,36 @@ else if (!params.h5File.exists())
                 }
 
                 hashingCalls = CellHashingService.get().generateHashingCallsForRawMatrix(parentReadset, output, ctx, params, existingCountMatrixUmiDir);
+
+                File metricFile = CellHashingService.get().getMetricsFile(hashingCalls);
+                if (!metricFile.exists())
+                {
+                    throw new PipelineJobException("Unable to find metrics file: " + metricFile.getPath());
+                }
+
+                try (CSVReader reader = new CSVReader(Readers.getReader(metricFile), '\t'))
+                {
+                    String[] line;
+                    while ((line = reader.readNext()) != null)
+                    {
+                        if ("HTOsDroppedAboveMinRetained".equals(line[1]))
+                        {
+                            String msg = "Hashing data contained HTOs with counts higher than those in the whitelist, this could indicate an error in the expected barcodes: " + line[2];
+                            if (params.failIfUnexpectedHtosFound)
+                            {
+                                throw new PipelineJobException(msg);
+                            }
+                            else
+                            {
+                                ctx.getLogger().warn(msg);
+                            }
+                        }
+                    }
+                }
+                catch (IOException e)
+                {
+                    throw new PipelineJobException(e);
+                }
             }
             else if (htosPerReadset.size() == 1)
             {