Bugfix feature barcoding and add metrics capture

Author: bbimber

singlecell/src/org/labkey/singlecell/run/CellRangerFeatureBarcodeHandler.java

@@ -5,10 +5,20 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.json.JSONObject;
+import org.labkey.api.collections.CaseInsensitiveHashMap;
+import org.labkey.api.data.CompareType;
+import org.labkey.api.data.ConvertHelper;
+import org.labkey.api.data.DbSchema;
+import org.labkey.api.data.DbSchemaType;
+import org.labkey.api.data.SimpleFilter;
+import org.labkey.api.data.Table;
+import org.labkey.api.data.TableInfo;
+import org.labkey.api.data.TableSelector;
 import org.labkey.api.module.ModuleLoader;
 import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.pipeline.RecordedAction;
+import org.labkey.api.query.FieldKey;
 import org.labkey.api.reader.Readers;
 import org.labkey.api.sequenceanalysis.SequenceOutputFile;
 import org.labkey.api.sequenceanalysis.model.Readset;
@@ -20,6 +30,7 @@
 import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor;
 import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper;
 import org.labkey.api.util.FileUtil;
+import org.labkey.api.util.PageFlowUtil;
 import org.labkey.api.writer.PrintWriters;
 import org.labkey.singlecell.CellHashingServiceImpl;
 import org.labkey.singlecell.SingleCellModule;
@@ -31,6 +42,7 @@
 import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 
 public class CellRangerFeatureBarcodeHandler extends AbstractParameterizedOutputHandler<SequenceOutputHandler.SequenceReadsetProcessor>
 {
@@ -164,8 +176,8 @@ else if (rs.getApplication().equals("CITE-Seq"))
             List<Pair<File, File>> inputFastqs = new ArrayList<>();
             rs.getReadData().forEach(rd -> {
                 inputFastqs.add(Pair.of(rd.getFile1(), rd.getFile2()));
-                output.addIntermediateFile(rd.getFile1(), "Input FASTQ");
-                output.addIntermediateFile(rd.getFile2(), "Input FASTQ");
+                action.addInputIfNotPresent(rd.getFile1(), "Input FASTQ");
+                action.addInputIfNotPresent(rd.getFile2(), "Input FASTQ");
             });
 
             List<String> args = wrapper.prepareCountArgs(output, id, ctx.getOutputDir(), rs, inputFastqs, extraArgs, false);
@@ -366,6 +378,112 @@ private File createFeatureRefForHashing(File outputDir, File hashingWhitelist) t
             return featuresCsv;
         }
 
+        @Override
+        public void complete(PipelineJob job, List<Readset> readsets, List<SequenceOutputFile> outputsCreated) throws PipelineJobException
+        {
+            if (outputsCreated.isEmpty())
+            {
+                job.getLogger().error("Expected outputs to be created");
+                return;
+            }
+
+            SequenceOutputFile so = null;
+            for (SequenceOutputFile o : outputsCreated)
+            {
+                if ("10x Run Summary".equals(o.getCategory()))
+                {
+                    continue;
+                }
+
+                so = o;
+                break;
+            }
+
+            if (so == null)
+            {
+                throw new PipelineJobException("Unable to find count matrix as output");
+            }
+
+            Readset rs = readsets.get(0);
+
+            File metrics = new File(so.getFile().getParentFile(), "metrics_summary.csv");
+            if (metrics.exists())
+            {
+                job.getLogger().debug("adding 10x metrics");
+                try (CSVReader reader = new CSVReader(Readers.getReader(metrics)))
+                {
+                    String[] line;
+                    String[] header = null;
+                    String[] metricValues = null;
 
+                    int i = 0;
+                    while ((line = reader.readNext()) != null)
+                    {
+                        if (i == 0)
+                        {
+                            header = line;
+                        }
+                        else
+                        {
+                            metricValues = line;
+                            break;
+                        }
+
+                        i++;
+                    }
+
+                    TableInfo ti = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("quality_metrics");
+
+                    //NOTE: if this job errored and restarted, we may have duplicate records:
+                    SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), so.getReadset());
+                    filter.addCondition(FieldKey.fromString("dataid"), so.getDataId(), CompareType.EQUAL);
+                    filter.addCondition(FieldKey.fromString("category"), rs.getApplication(), CompareType.EQUAL);
+                    filter.addCondition(FieldKey.fromString("container"), job.getContainer().getId(), CompareType.EQUAL);
+                    TableSelector ts = new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null);
+                    if (ts.exists())
+                    {
+                        job.getLogger().info("Deleting existing QC metrics (probably from prior restarted job)");
+                        ts.getArrayList(Integer.class).forEach(rowid -> {
+                            Table.delete(ti, rowid);
+                        });
+                    }
+
+                    for (int j = 0; j < header.length; j++)
+                    {
+                        Map<String, Object> toInsert = new CaseInsensitiveHashMap<>();
+                        toInsert.put("container", job.getContainer().getId());
+                        toInsert.put("createdby", job.getUser().getUserId());
+                        toInsert.put("created", new Date());
+                        toInsert.put("readset", rs.getReadsetId());
+                        toInsert.put("dataid", so.getDataId());
+
+                        toInsert.put("category", "Cell Ranger");
+                        toInsert.put("metricname", header[j]);
+
+                        metricValues[j] = metricValues[j].replaceAll(",", "");
+                        Object val = metricValues[j];
+                        if (metricValues[j].contains("%"))
+                        {
+                            metricValues[j] = metricValues[j].replaceAll("%", "");
+                            Double d = ConvertHelper.convert(metricValues[j], Double.class);
+                            d = d / 100.0;
+                            val = d;
+                        }
+
+                        toInsert.put("metricvalue", val);
+
+                        Table.insert(job.getUser(), ti, toInsert);
+                    }
+                }
+                catch (IOException e)
+                {
+                    throw new PipelineJobException(e);
+                }
+            }
+            else
+            {
+                job.getLogger().warn("unable to find metrics file: " + metrics.getPath());
+            }
+        }
     }
 }