Merge pull request #145 from LabKey/fb_merge_22.3_to_develop

bbimber · web-flow · commit 757eafcde37d · 2022-05-26T08:37:15.000-07:00
Merge discvr-22.3 to develop
diff --git a/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java b/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java
@@ -348,38 +348,52 @@ public static void executeR(SequenceOutputHandler.JobContext ctx, String dockerC
         localBashScript.delete();
     }
 
-    protected String prepareValueForR(SeuratToolParameter pd)
+    protected void addParameterVariables(SeuratToolParameter pd, List<String> body)
     {
         String val = StringUtils.trimToNull(pd.extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx()));
         if (val == null)
         {
-            return "NULL";
+            body.add((pd.getVariableName() + " <- NULL"));
         }
         else if ("false".equals(val))
         {
-            return "FALSE";
+            body.add((pd.getVariableName() + " <- FALSE"));
         }
         else if ("true".equals(val))
         {
-            return "TRUE";
+            body.add((pd.getVariableName() + " <- TRUE"));
         }
         else if (NumberUtils.isCreatable(val))
         {
-            return val;
+            body.add((pd.getVariableName() + " <- " + val));
         }
         else if ("sequenceanalysis-trimmingtextarea".equals(pd.getFieldXtype()))
         {
             val = val.replace("'", "\\\'");
-            String[] vals = val.split(pd.getDelimiter());
-            return "c('" + StringUtils.join(vals, "','") + "')";
+            serializeMultiValueParam(pd, body, val);
         }
         else if (pd.isMultiValue())
         {
-            String[] vals = val.split(pd.getDelimiter());
-            return "c('" + StringUtils.join(vals, "','") + "')";
+            serializeMultiValueParam(pd, body, val);
         }
+        else
+        {
+            body.add((pd.getVariableName() + " <- '" + val + "'"));
+        }
+    }
 
-        return  "'" + val + "'";
+    private void serializeMultiValueParam(SeuratToolParameter pd, List<String> body, String val)
+    {
+        String[] vals = val.split(pd.getDelimiter());
+        final int batchSize = 75;
+        int numBatches = (int)Math.ceil((double)vals.length / batchSize);
+
+        for (int i=0;i<numBatches;i++)
+        {
+            int start = (i * batchSize);
+            int end = Math.min(start + batchSize, vals.length);
+            body.add(pd.getVariableName() + " <- " + "c(" + (i == 0 ? "" : pd.getVariableName() + ", ") + "'" + StringUtils.join(Arrays.copyOfRange(vals, start, end), "','") + "')");
+        }
     }
 
     protected List<String> loadChunkFromFile() throws PipelineJobException
@@ -423,7 +437,7 @@ protected Chunk createParamChunk(SequenceOutputHandler.JobContext ctx, List<Seur
                 SeuratToolParameter stp = (SeuratToolParameter)pd;
                 if (stp.shouldIncludeInMarkdown(getPipelineCtx().getJob(), getProvider(), getStepIdx()))
                 {
-                    body.add(((SeuratToolParameter) pd).getVariableName() + " <- " + prepareValueForR(stp));
+                    addParameterVariables(stp, body);
                 }
             }
         }
diff --git a/singlecell/api-src/org/labkey/api/singlecell/pipeline/SeuratToolParameter.java b/singlecell/api-src/org/labkey/api/singlecell/pipeline/SeuratToolParameter.java
@@ -50,7 +50,7 @@ public String getVariableName()
         return _rName == null ? getName() : _rName;
     }
 
-    public boolean shouldIncludeInMarkdown(PipelineJob job, PipelineStepProvider provider, int stepIdx)
+    public boolean shouldIncludeInMarkdown(PipelineJob job, PipelineStepProvider<?> provider, int stepIdx)
     {
         if (!_includeIfEmptyOrNull)
         {
diff --git a/singlecell/resources/chunks/AppendNimble.R b/singlecell/resources/chunks/AppendNimble.R
diff --git a/singlecell/resources/chunks/TcrFilter.R b/singlecell/resources/chunks/TcrFilter.R
@@ -18,7 +18,11 @@ for (datasetId in names(seuratObjects)) {
 
     cdr3ForLocus <- gsub(cdr3ForLocus, pattern = paste0(fieldName, ':'), replacement = '')
     matchingCells <- sapply(seuratObj@meta.data[[fieldName]], function(x){
-      values <- unlist(strsplit(x, split = ','))
+      if (is.na(x)) {
+        return(FALSE)
+      }
+
+      values <- unlist(strsplit(as.character(x), split = ','))
       return(length(intersect(values, cdr3ForLocus)) != 0)
     })
 
@@ -37,8 +41,9 @@ for (datasetId in names(seuratObjects)) {
   if (all(is.null(cellsToKeep))) {
     print('There were no matching cells')
   } else {
+    print(paste0('Total passing cells: ', length(cellsToKeep)))
     seuratObj <- subset(seuratObj, cells = cellsToKeep)
-    #saveData(seuratObj, datasetId)
+    saveData(seuratObj, datasetId)
     totalPassed <- totalPassed + 1
   }
 
diff --git a/singlecell/src/org/labkey/singlecell/SingleCellModule.java b/singlecell/src/org/labkey/singlecell/SingleCellModule.java
@@ -188,7 +188,7 @@ public static void registerPipelineSteps()
         SequencePipelineService.get().registerPipelineStep(new CheckExpectations.Provider());
         SequencePipelineService.get().registerPipelineStep(new CommonFilters.Provider());
         SequencePipelineService.get().registerPipelineStep(new RunVision.Provider());
-        SequencePipelineService.get().registerPipelineStep(new NimbleAppend.Provider());
+        SequencePipelineService.get().registerPipelineStep(new AppendNimble.Provider());
         SequencePipelineService.get().registerPipelineStep(new AppendTcr.Provider());
         SequencePipelineService.get().registerPipelineStep(new TcrFilter.Provider());
 
diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendNimble.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendNimble.java
@@ -15,9 +15,9 @@
 import java.util.List;
 import java.util.Set;
 
-public class NimbleAppend extends AbstractRDiscvrStep
+public class AppendNimble extends AbstractRDiscvrStep
 {
-    public NimbleAppend(PipelineContext ctx, NimbleAppend.Provider provider)
+    public AppendNimble(PipelineContext ctx, AppendNimble.Provider provider)
     {
         super(provider, ctx);
     }
@@ -35,9 +35,9 @@ public Provider()
 
 
         @Override
-        public NimbleAppend create(PipelineContext ctx)
+        public AppendNimble create(PipelineContext ctx)
         {
-            return new NimbleAppend(ctx, this);
+            return new AppendNimble(ctx, this);
         }
     }
 
@@ -62,6 +62,7 @@ protected Chunk createParamChunk(SequenceOutputHandler.JobContext ctx, List<Seur
         ret.bodyLines.add("nimbleGenomes <- list(");
         String genomeStr = getProvider().getParameterByName("nimbleGenomes").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class);
         JSONArray json = new JSONArray(genomeStr);
+        String delim = "";
         for (int i = 0; i < json.length(); i++)
         {
             JSONArray arr = json.getJSONArray(i);
@@ -72,7 +73,8 @@ protected Chunk createParamChunk(SequenceOutputHandler.JobContext ctx, List<Seur
 
             int genomeId = arr.getInt(0);
             String targetAssay = arr.getString(1);
-            ret.bodyLines.add("\t" + genomeId + " = " + targetAssay);
+            ret.bodyLines.add("\t" + delim + "'" + genomeId + "' = '" + targetAssay + "'");
+            delim = ",";
         }
         ret.bodyLines.add(")");
 
diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CheckExpectations.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CheckExpectations.java
@@ -72,10 +72,15 @@ protected Chunk createParamChunk(SequenceOutputHandler.JobContext ctx, List<Seur
 
         for (SeuratObjectWrapper so : inputObjects)
         {
+            if (so.getSequenceOutputFile().getReadset() == null)
+            {
+                throw new PipelineJobException("Unable to find readset for outputfile: " + so.getSequenceOutputFileId() + ". This set requires single-dataset inputs. Removing this step may be a solution.");
+            }
+
             Readset parentReadset = ctx.getSequenceSupport().getCachedReadset(so.getSequenceOutputFile().getReadset());
             if (parentReadset == null)
             {
-                throw new PipelineJobException("Unable to find readset for outputfile: " + so.getSequenceOutputFileId());
+                throw new PipelineJobException("Unable to find readset for outputfile: " + so.getSequenceOutputFileId() + ". This set requires single-dataset inputs. Removing this step may be a solution.");
             }
 
             Set<String> htosPerReadset = CellHashingServiceImpl.get().getHtosForParentReadset(parentReadset.getReadsetId(), ctx.getSourceDirectory(), ctx.getSequenceSupport(), false);

Original file line number	Diff line number	Diff line change
`@@ -348,38 +348,52 @@ public static void executeR(SequenceOutputHandler.JobContext ctx, String dockerC`
`348`	`348`	`localBashScript.delete();`
`349`	`349`	`}`
`350`	`350`
`351`		`- protected String prepareValueForR(SeuratToolParameter pd)`
	`351`	`+ protected void addParameterVariables(SeuratToolParameter pd, List<String> body)`
`352`	`352`	`{`
`353`	`353`	`String val = StringUtils.trimToNull(pd.extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx()));`
`354`	`354`	`if (val == null)`
`355`	`355`	`{`
`356`		`- return "NULL";`
	`356`	`+ body.add((pd.getVariableName() + " <- NULL"));`
`357`	`357`	`}`
`358`	`358`	`else if ("false".equals(val))`
`359`	`359`	`{`
`360`		`- return "FALSE";`
	`360`	`+ body.add((pd.getVariableName() + " <- FALSE"));`
`361`	`361`	`}`
`362`	`362`	`else if ("true".equals(val))`
`363`	`363`	`{`
`364`		`- return "TRUE";`
	`364`	`+ body.add((pd.getVariableName() + " <- TRUE"));`
`365`	`365`	`}`
`366`	`366`	`else if (NumberUtils.isCreatable(val))`
`367`	`367`	`{`
`368`		`- return val;`
	`368`	`+ body.add((pd.getVariableName() + " <- " + val));`
`369`	`369`	`}`
`370`	`370`	`else if ("sequenceanalysis-trimmingtextarea".equals(pd.getFieldXtype()))`
`371`	`371`	`{`
`372`	`372`	`val = val.replace("'", "\\\'");`
`373`		`- String[] vals = val.split(pd.getDelimiter());`
`374`		`- return "c('" + StringUtils.join(vals, "','") + "')";`
	`373`	`+ serializeMultiValueParam(pd, body, val);`
`375`	`374`	`}`
`376`	`375`	`else if (pd.isMultiValue())`
`377`	`376`	`{`
`378`		`- String[] vals = val.split(pd.getDelimiter());`
`379`		`- return "c('" + StringUtils.join(vals, "','") + "')";`
	`377`	`+ serializeMultiValueParam(pd, body, val);`
`380`	`378`	`}`
	`379`	`+ else`
	`380`	`+ {`
	`381`	`+ body.add((pd.getVariableName() + " <- '" + val + "'"));`
	`382`	`+ }`
	`383`	`+ }`
`381`	`384`
`382`		`- return "'" + val + "'";`
	`385`	`+ private void serializeMultiValueParam(SeuratToolParameter pd, List<String> body, String val)`
	`386`	`+ {`
	`387`	`+ String[] vals = val.split(pd.getDelimiter());`
	`388`	`+ final int batchSize = 75;`
	`389`	`+ int numBatches = (int)Math.ceil((double)vals.length / batchSize);`
	`390`	`+`
	`391`	`+ for (int i=0;i<numBatches;i++)`
	`392`	`+ {`
	`393`	`+ int start = (i * batchSize);`
	`394`	`+ int end = Math.min(start + batchSize, vals.length);`
	`395`	`+ body.add(pd.getVariableName() + " <- " + "c(" + (i == 0 ? "" : pd.getVariableName() + ", ") + "'" + StringUtils.join(Arrays.copyOfRange(vals, start, end), "','") + "')");`
	`396`	`+ }`
`383`	`397`	`}`
`384`	`398`
`385`	`399`	`protected List<String> loadChunkFromFile() throws PipelineJobException`
`@@ -423,7 +437,7 @@ protected Chunk createParamChunk(SequenceOutputHandler.JobContext ctx, List<Seur`
`423`	`437`	`SeuratToolParameter stp = (SeuratToolParameter)pd;`
`424`	`438`	`if (stp.shouldIncludeInMarkdown(getPipelineCtx().getJob(), getProvider(), getStepIdx()))`
`425`	`439`	`{`
`426`		`- body.add(((SeuratToolParameter) pd).getVariableName() + " <- " + prepareValueForR(stp));`
	`440`	`+ addParameterVariables(stp, body);`
`427`	`441`	`}`
`428`	`442`	`}`
`429`	`443`	`}`
Original file line number	Diff line number	Diff line change
`@@ -50,7 +50,7 @@ public String getVariableName()`
`50`	`50`	`return _rName == null ? getName() : _rName;`
`51`	`51`	`}`
`52`	`52`
`53`		`- public boolean shouldIncludeInMarkdown(PipelineJob job, PipelineStepProvider provider, int stepIdx)`
	`53`	`+ public boolean shouldIncludeInMarkdown(PipelineJob job, PipelineStepProvider<?> provider, int stepIdx)`
`54`	`54`	`{`
`55`	`55`	`if (!_includeIfEmptyOrNull)`
`56`	`56`	`{`
Original file line number	Diff line number	Diff line change
`@@ -15,9 +15,9 @@`
`15`	`15`	`import java.util.List;`
`16`	`16`	`import java.util.Set;`
`17`	`17`
`18`		`-public class NimbleAppend extends AbstractRDiscvrStep`
	`18`	`+public class AppendNimble extends AbstractRDiscvrStep`
`19`	`19`	`{`
`20`		`- public NimbleAppend(PipelineContext ctx, NimbleAppend.Provider provider)`
	`20`	`+ public AppendNimble(PipelineContext ctx, AppendNimble.Provider provider)`
`21`	`21`	`{`
`22`	`22`	`super(provider, ctx);`
`23`	`23`	`}`
`@@ -35,9 +35,9 @@ public Provider()`
`35`	`35`
`36`	`36`
`37`	`37`	`@Override`
`38`		`- public NimbleAppend create(PipelineContext ctx)`
	`38`	`+ public AppendNimble create(PipelineContext ctx)`
`39`	`39`	`{`
`40`		`- return new NimbleAppend(ctx, this);`
	`40`	`+ return new AppendNimble(ctx, this);`
`41`	`41`	`}`
`42`	`42`	`}`
`43`	`43`
`@@ -62,6 +62,7 @@ protected Chunk createParamChunk(SequenceOutputHandler.JobContext ctx, List<Seur`
`62`	`62`	`ret.bodyLines.add("nimbleGenomes <- list(");`
`63`	`63`	`String genomeStr = getProvider().getParameterByName("nimbleGenomes").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class);`
`64`	`64`	`JSONArray json = new JSONArray(genomeStr);`
	`65`	`+ String delim = "";`
`65`	`66`	`for (int i = 0; i < json.length(); i++)`
`66`	`67`	`{`
`67`	`68`	`JSONArray arr = json.getJSONArray(i);`
`@@ -72,7 +73,8 @@ protected Chunk createParamChunk(SequenceOutputHandler.JobContext ctx, List<Seur`
`72`	`73`
`73`	`74`	`int genomeId = arr.getInt(0);`
`74`	`75`	`String targetAssay = arr.getString(1);`
`75`		`- ret.bodyLines.add("\t" + genomeId + " = " + targetAssay);`
	`76`	`+ ret.bodyLines.add("\t" + delim + "'" + genomeId + "' = '" + targetAssay + "'");`
	`77`	`+ delim = ",";`
`76`	`78`	`}`
`77`	`79`	`ret.bodyLines.add(")");`
`78`	`80`
Original file line number	Diff line number	Diff line change
`@@ -72,10 +72,15 @@ protected Chunk createParamChunk(SequenceOutputHandler.JobContext ctx, List<Seur`
`72`	`72`
`73`	`73`	`for (SeuratObjectWrapper so : inputObjects)`
`74`	`74`	`{`
	`75`	`+ if (so.getSequenceOutputFile().getReadset() == null)`
	`76`	`+ {`
	`77`	`+ throw new PipelineJobException("Unable to find readset for outputfile: " + so.getSequenceOutputFileId() + ". This set requires single-dataset inputs. Removing this step may be a solution.");`
	`78`	`+ }`
	`79`	`+`
`75`	`80`	`Readset parentReadset = ctx.getSequenceSupport().getCachedReadset(so.getSequenceOutputFile().getReadset());`
`76`	`81`	`if (parentReadset == null)`
`77`	`82`	`{`
`78`		`- throw new PipelineJobException("Unable to find readset for outputfile: " + so.getSequenceOutputFileId());`
	`83`	`+ throw new PipelineJobException("Unable to find readset for outputfile: " + so.getSequenceOutputFileId() + ". This set requires single-dataset inputs. Removing this step may be a solution.");`
`79`	`84`	`}`
`80`	`85`
`81`	`86`	`Set<String> htosPerReadset = CellHashingServiceImpl.get().getHtosForParentReadset(parentReadset.getReadsetId(), ctx.getSourceDirectory(), ctx.getSequenceSupport(), false);`