Merge discvr-19.3 to develop

Author: bbimber

.github/workflows/sync-repos.yml

@@ -4,6 +4,8 @@ on:
     - cron:  "10 */3 * * *"
 jobs:
   sync-release-branches:
+    # github.head_ref should only be set on forked repos, meaning this action should only run on the primary fork
+    if: github.head_ref == ''
     runs-on: ubuntu-latest
     steps:
       - name: "Sync Release Branches"
@@ -15,6 +17,7 @@ jobs:
           destination_branch_prefix: "discvr-"
           github_token: ${{ secrets.GITHUB_TOKEN }}
   sync-develop:
+    if: github.head_ref == ''
     runs-on: ubuntu-latest
     steps:
       - name: "Sync Develop Branch"

SequenceAnalysis/resources/external/scRNAseq/htoClassifier.Rmd

@@ -2,10 +2,19 @@
 title: 'Cell Hashing'
 ---
 
-```{r}
+```{r setup}
 
 library(OOSAP)
 
+cores <- Sys.getenv('SEQUENCEANALYSIS_MAX_THREADS')
+if (cores != ''){
+    print(paste0('Setting threads to ', cores))
+    future::plan("multiprocess", workers = as.integer(cores))
+    Sys.setenv('OMP_NUM_THREADS' = cores)
+} else {
+    print('SEQUENCEANALYSIS_MAX_THREADS not set, will not set cores')
+}
+
 ```
 
 ## Basic QC and Filtering on input:

SequenceAnalysis/resources/external/scRNAseq/htoClassifier.sh

@@ -19,11 +19,18 @@ if [ $# -ge 8 ];then
     WHITELIST="whitelistFile<-'"${8}"';"
 fi
 
+ENV_OPTS=""
 RAM_OPTS=""
 if [ ! -z $SEQUENCEANALYSIS_MAX_RAM ];then
     RAM_OPTS=" --memory=${SEQUENCEANALYSIS_MAX_RAM}g"
+
+    ENV_OPTS=" -e SEQUENCEANALYSIS_MAX_RAM"
+fi
+
+if [ ! -z SEQUENCEANALYSIS_MAX_THREADS ];then
+    ENV_OPTS=${ENV_OPTS}" -e SEQUENCEANALYSIS_MAX_THREADS="${SEQUENCEANALYSIS_MAX_THREADS}
 fi
 
 sudo $DOCKER pull bimberlab/oosap
 
-sudo $DOCKER run --rm=true $RAM_OPTS -v "${WD}:/work" -v "${HOME}:/homeDir" -u $UID -e USERID=$UID -w /work -e HOME=/homeDir bimberlab/oosap Rscript -e "barcodeDir <- '"${CITESEQ_COUNT_DIR}"';finalCallFile <- '"${FINAL_CALLS}"';doHtoFilter <- "${DO_HTO_FILTER}";maxValueForColSumFilter <- "${MIN_READS_PER_CELL}";allCallsOutFile <- '"${RAW_CALLS}"';metricsFile <- '"${METRICS_FILE}"';"${WHITELIST}"rmarkdown::render('htoClassifier.Rmd', output_file = '"${HTML_FILE}"')"
+sudo $DOCKER run --rm=true $RAM_OPTS $ENV_OPTS -v "${WD}:/work" -v "${HOME}:/homeDir" -u $UID -e USERID=$UID -w /work -e HOME=/homeDir bimberlab/oosap Rscript -e "barcodeDir <- '"${CITESEQ_COUNT_DIR}"';finalCallFile <- '"${FINAL_CALLS}"';doHtoFilter <- "${DO_HTO_FILTER}";maxValueForColSumFilter <- "${MIN_READS_PER_CELL}";allCallsOutFile <- '"${RAW_CALLS}"';metricsFile <- '"${METRICS_FILE}"';"${WHITELIST}"rmarkdown::render('htoClassifier.Rmd', output_file = '"${HTML_FILE}"')"

SequenceAnalysis/resources/web/SequenceAnalysis/panel/AlignmentAnalysisPanel.js

@@ -140,7 +140,7 @@ Ext4.define('SequenceAnalysis.panel.AlignmentAnalysisPanel', {
         };
 
         //first add the general params
-        Ext4.apply(json, this.down('#runInformation').getForm().getValues());
+        Ext4.apply(json, this.down('#runInformation').getForm().getFieldValues());
 
         //and sample information
         json.analysisIds = this.analyses;

SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceAnalysisPanel.js

@@ -256,7 +256,7 @@ Ext4.define('SequenceAnalysis.panel.SequenceAnalysisPanel', {
         };
 
         //first add the general params
-        Ext4.apply(json, this.down('#runInformation').getForm().getValues());
+        Ext4.apply(json, this.down('#runInformation').getForm().getFieldValues());
 
         json['alignment.doAlignment'] = this.down('#doAlignment').getValue();
 

SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js

@@ -453,7 +453,7 @@ Ext4.define('SequenceAnalysis.panel.VariantProcessingPanel', {
 		};
 
 		//first add the general params
-		Ext4.apply(json, this.down('#runInformation').getForm().getValues());
+		Ext4.apply(json, this.down('#runInformation').getForm().getFieldValues());
 
 		//then append each section
 		var sections = this.query('sequenceanalysis-analysissectionpanel');

SequenceAnalysis/resources/web/SequenceAnalysis/window/OutputHandlerWindow.js

@@ -250,7 +250,8 @@ Ext4.define('SequenceAnalysis.window.OutputHandlerWindow', {
             return;
         }
 
-        var params = this.down('form').getForm().getValues();
+        //Note: use getFieldValues() to also include unchecked checkboxes
+        var params = this.down('form').getForm().getFieldValues();
         var json = {
             handlerType: this.handlerType,
             handlerClass: this.handlerClass,

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java

@@ -3659,6 +3659,21 @@ else if (fastaFileType.isType(data.getFile()))
                             }
                         }
                     }
+
+                    File dict = new File(data.getFile().getParentFile(), FileUtil.getBaseName(data.getFile()) + ".dict");
+                    if (dict.exists())
+                    {
+                        files.add(dict);
+                    }
+                    else
+                    {
+                        //Try alternate name scheme
+                        dict = new File(data.getFile() + ".dict");
+                        if (dict.exists())
+                        {
+                            files.add(dict);
+                        }
+                    }
                 }
             }
 

SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/CellHashingHandler.java

@@ -85,6 +85,7 @@ public static List<ToolParameterDescriptor> getDefaultParams()
                     put("checked", true);
                 }}, true),
                 ToolParameterDescriptor.create("editDistance", "Edit Distance", null, "ldk-integerfield", null, 1),
+                ToolParameterDescriptor.create("excludeFailedcDNA", "Exclude Failed cDNA", "If selected, cDNAs with non-blank status fields will be omitted", "checkbox", null, true),
                 ToolParameterDescriptor.create("minCountPerCell", "Min Reads/Cell", null, "ldk-integerfield", null, 5),
                 ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("-cells"), "cells", "Expected Cells", null, "ldk-integerfield", null, 20000),
                 ToolParameterDescriptor.create("tagGroup", "Tag List", null, "ldk-simplelabkeycombo", new JSONObject(){{
@@ -276,6 +277,7 @@ private Map<String, Object> executeCiteSeqCount(JobContext ctx, RecordedAction a
         CiteSeqCountWrapper wrapper = new CiteSeqCountWrapper(ctx.getLogger());
         ReadData rd = rs.getReadData().get(0);
 
+        //ctx.getJob().setStatus(PipelineJob.TaskStatus.running, "Running CITE-seq-count with edit distance: " + editDistance);
         List<String> args = new ArrayList<>();
 
         args.addAll(getClientCommandArgs(ctx.getParams()));

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java

@@ -15,6 +15,8 @@
  */
 package org.labkey.sequenceanalysis.pipeline;
 
+import htsjdk.samtools.SAMSequenceDictionary;
+import htsjdk.variant.utils.SAMSequenceDictionaryExtractor;
 import org.apache.commons.io.FileUtils;
 import org.jetbrains.annotations.NotNull;
 import org.labkey.api.collections.CaseInsensitiveHashMap;
@@ -193,6 +195,33 @@ public RecordedActionSet run() throws PipelineJobException
                     File dict = new File(existingFasta.getFile().getParentFile(), basename + ".dict");
                     if (dict.exists())
                     {
+                        SAMSequenceDictionary extractor = SAMSequenceDictionaryExtractor.extractDictionary(dict.toPath());
+
+                        //ensure set of sequence names is identical:
+                        List<String> dictSequenceNames = new ArrayList<>();
+                        extractor.getSequences().forEach(x -> dictSequenceNames.add(x.getSequenceName()));
+
+                        Map<String, ReferenceLibraryMember> libraryMemberMap = new HashMap<>();
+                        libraryMembers.forEach(x -> libraryMemberMap.put(x.getHeaderName(), x));
+                        List<String> uniqueIncomingSequenceNames = new ArrayList<>(libraryMemberMap.keySet());
+                        uniqueIncomingSequenceNames.removeAll(dictSequenceNames);
+
+                        if (libraryMemberMap.size() == dictSequenceNames.size() && uniqueIncomingSequenceNames.isEmpty())
+                        {
+                            getJob().getLogger().info("Sorting output to match pre-existing dictionary file");
+
+                            List<ReferenceLibraryMember> orderedMembers = new ArrayList<>();
+                            extractor.getSequences().forEach(x -> {
+                                orderedMembers.add(libraryMemberMap.get(x.getSequenceName()));
+                            });
+
+                            libraryMembers = orderedMembers;
+                        }
+                        else
+                        {
+                            getJob().getLogger().info("An existing dictionary file was found, but the set of sequences differs.  Will not use.");
+                        }
+
                         dict.delete();
                     }