Merge pull request #83 from LabKey/fb_merge_discvr-20.11

Merge discvr-20.11 to develop

Author: bbimber

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/ReferenceGenomeManager.java

@@ -1,6 +1,7 @@
 package org.labkey.api.sequenceanalysis.pipeline;
 
-import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.SystemUtils;
 import org.apache.logging.log4j.Logger;
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper;
@@ -60,11 +61,32 @@ private boolean isUpToDate(ReferenceGenome genome)
         return lastSync >= lastUpdated;
     }
 
-    public void markGenomeModified(ReferenceGenome genome, Logger log) throws IOException
+    public void markGenomeModified(ReferenceGenome genome, Logger log) throws PipelineJobException
     {
         File toUpdate = getLocalUpdateFile(genome);
         log.info("Marking genome as modified: " + toUpdate.getPath());
-        Files.touch(toUpdate);
+        touchFile(toUpdate, log);
+    }
+
+    //NOTE: Java implementations of touch are erroring between the cluster and NFS filesystem
+    private void touchFile(File target, Logger log) throws PipelineJobException
+    {
+        if (SystemUtils.IS_OS_WINDOWS)
+        {
+            try
+            {
+                FileUtils.touch(target);
+            }
+            catch (IOException e)
+            {
+                throw new PipelineJobException(e);
+            }
+        }
+        else
+        {
+            SimpleScriptWrapper wrapper = new SimpleScriptWrapper(log);
+            wrapper.execute(Arrays.asList("/bin/bash", "-c", "$(which touch) '" + target.getPath() + "'"));
+        }
     }
 
     public void cacheGenomeLocally(ReferenceGenome genome, Logger log) throws PipelineJobException
@@ -80,37 +102,32 @@ public void cacheGenomeLocally(ReferenceGenome genome, Logger log) throws Pipeli
             return;
         }
 
+        File localCacheDir = SequencePipelineService.get().getRemoteGenomeCacheDirectory();
         if (isUpToDate(genome))
         {
             log.debug("Genome up-to-date, will not repeat rsync");
+            genome.setWorkingFasta(new File(new File(localCacheDir, genome.getGenomeId().toString()), genome.getSourceFastaFile().getName()));
+
             return;
         }
 
-        File localCacheDir = SequencePipelineService.get().getRemoteGenomeCacheDirectory();
         log.info("attempting to rsync genome to local disks: " + localCacheDir.getPath());
 
         File sourceDir = genome.getSourceFastaFile().getParentFile();
 
         //Note: neither source nor dest have trailing slashes, so the entire source (i.e '128', gets synced into a subdir of dest)
         new SimpleScriptWrapper(log).execute(Arrays.asList(
-                "rsync", "-r", "-a", "--delete", "--delete-excluded", "--no-owner", "--no-group", sourceDir.getPath(), localCacheDir.getPath()
+                "rsync", "-r", "-a", "--delete", "--no-owner", "--no-group", sourceDir.getPath(), localCacheDir.getPath()
         ));
 
-        try
-        {
-            File lastUpdate = getLocalUpdateFile(genome);
-            if (!lastUpdate.exists())
-            {
-                Files.touch(lastUpdate);
-            }
-
-            Files.touch(getRemoteSyncFile(genome.getGenomeId()));
-        }
-        catch (IOException e)
+        File lastUpdate = getLocalUpdateFile(genome);
+        if (!lastUpdate.exists())
         {
-            throw new PipelineJobException(e);
+            touchFile(lastUpdate, log);
         }
 
+        touchFile(getRemoteSyncFile(genome.getGenomeId()), log);
+
         genome.setWorkingFasta(new File(new File(localCacheDir, genome.getGenomeId().toString()), genome.getSourceFastaFile().getName()));
     }
 }

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/RCommandWrapper.java

@@ -70,7 +70,7 @@ private String inferRPath(Logger log)
         //preferentially use R config setup in scripting props.  only works if running locally.
         if (PipelineJobService.get().getLocationType() == PipelineJobService.LocationType.WebServer)
         {
-            LabKeyScriptEngineManager svc = LabKeyScriptEngineManager.get();
+            LabKeyScriptEngineManager svc = ServiceRegistry.get().getService(LabKeyScriptEngineManager.class);
             for (ExternalScriptEngineDefinition def : svc.getEngineDefinitions())
             {
                 if (RScriptEngineFactory.isRScriptEngine(def.getExtensions()))

SequenceAnalysis/resources/external/scRNAseq/htoClassifier.Rmd

@@ -1,11 +1,15 @@
 ---
 title: 'Cell Hashing'
+output: html_document
+
 ---
 
 ```{r setup}
 
 library(OOSAP)
 
+knitr::opts_chunk$set(message=FALSE, warning=FALSE, echo=TRUE, error = TRUE)
+
 cores <- Sys.getenv('SEQUENCEANALYSIS_MAX_THREADS')
 if (cores != ''){
     print(paste0('Setting threads to ', cores))
@@ -15,6 +19,15 @@ if (cores != ''){
     print('SEQUENCEANALYSIS_MAX_THREADS not set, will not set cores')
 }
 
+print('Global variables: ')
+for (v in c('doHtoFilter', 'maxValueForColSumFilter', 'useSeurat', 'useMultiSeq', 'whitelistFile')){
+    if (exists(v)){
+        print(paste0(v, ': ', get(v)))
+    } else {
+      print(paste0(v, ': not defined'))
+    }
+}
+
 ```
 
 ## Basic QC and Filtering on input:

SequenceAnalysis/resources/external/scRNAseq/htoClassifier.sh

@@ -36,4 +36,6 @@ fi
 
 sudo $DOCKER pull bimberlab/oosap
 
-sudo $DOCKER run --rm=true $RAM_OPTS $ENV_OPTS -v "${WD}:/work" -v "${HOME}:/homeDir" -u $UID -e USERID=$UID -w /work -e HOME=/homeDir bimberlab/oosap Rscript -e "barcodeDir <- '"${CITESEQ_COUNT_DIR}"';finalCallFile <- '"${FINAL_CALLS}"';doHtoFilter <- "${DO_HTO_FILTER}";maxValueForColSumFilter <- "${MIN_READS_PER_CELL}";allCallsOutFile <- '"${RAW_CALLS}"';metricsFile <- '"${METRICS_FILE}"';useSeurat <- ${USE_SEURAT};useMultiSeq <- ${USE_MULTISEQ};"${WHITELIST}"rmarkdown::render('htoClassifier.Rmd', output_file = '"${HTML_FILE}"')"
+sudo $DOCKER run --rm=true $RAM_OPTS $ENV_OPTS -v "${WD}:/work" -v "${HOME}:/homeDir" -u $UID -e USERID=$UID -w /work -e HOME=/homeDir bimberlab/oosap Rscript -e "barcodeDir <- '"${CITESEQ_COUNT_DIR}"';finalCallFile <- '"${FINAL_CALLS}"';doHtoFilter <- "${DO_HTO_FILTER}";maxValueForColSumFilter <- "${MIN_READS_PER_CELL}";allCallsOutFile <- '"${RAW_CALLS}"';metricsFile <- '"${METRICS_FILE}"';useSeurat <- ${USE_SEURAT};useMultiSeq <- ${USE_MULTISEQ};"${WHITELIST}"rmarkdown::render('htoClassifier.Rmd', clean=TRUE, output_format = 'html_document', output_file = '"${HTML_FILE}"');print('Rmarkdown complete');"
+
+echo 'script complete'

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java

@@ -305,6 +305,7 @@ private void processContainer(Container c, Logger log) throws IOException
                         expectedChildren.add("alignerIndexes");
                         expectedChildren.add("tracks");
                         expectedChildren.add("chainFiles");
+                        expectedChildren.add(".lastUpdate");
 
                         for (String fileName : child.list())
                         {

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisManager.java

@@ -55,6 +55,7 @@
 import org.labkey.api.query.FieldKey;
 import org.labkey.api.query.InvalidKeyException;
 import org.labkey.api.query.QueryService;
+import org.labkey.api.query.QueryUpdateService;
 import org.labkey.api.query.QueryUpdateServiceException;
 import org.labkey.api.query.UserSchema;
 import org.labkey.api.reader.FastaDataLoader;
@@ -448,6 +449,8 @@ private void deleteRefNtSequence(User user, Container container, List<Integer> r
 
         try (DbScope.Transaction transaction = s.getSchema().getScope().ensureTransaction())
         {
+            List<Map<String, Object>> toDeleteQus = new ArrayList<>();
+
             for (int rowId : rowIds)
             {
                 //first data from analyses
@@ -487,13 +490,28 @@ private void deleteRefNtSequence(User user, Container container, List<Integer> r
                 {
                     Map<String, Object> map = new CaseInsensitiveHashMap<>();
                     map.put("rowid", rowId);
-                    List<Map<String, Object>> toDelete = Arrays.asList(map);
+                    toDeleteQus.add(map);
+                }
+            }
 
-                    Map<String, Object> scriptContext = new HashMap<>();
-                    scriptContext.put("deleteFromServer", true);  //a flag to make the trigger script accept this
-                    us.getTable(SequenceAnalysisSchema.TABLE_REF_NT_SEQUENCES).getUpdateService().deleteRows(user, container, toDelete, null, scriptContext);
+            if (!toDeleteQus.isEmpty())
+            {
+                Map<String, Object> scriptContext = new HashMap<>();
+                scriptContext.put("deleteFromServer", true);  //a flag to make the trigger script accept this
+
+                QueryUpdateService qus = us.getTable(SequenceAnalysisSchema.TABLE_REF_NT_SEQUENCES).getUpdateService();
+
+                int batchSize = 2500;
+                int numBatches = (int)Math.ceil(toDeleteQus.size() / (double)batchSize);
+
+                for (int i = 0; i < numBatches; i++)
+                {
+                    int start = i * batchSize;
+                    List<Map<String, Object>> subset = toDeleteQus.subList(start, Math.min(toDeleteQus.size(), start + batchSize));
+                    qus.deleteRows(user, container, subset, null, scriptContext);
                 }
             }
+
             transaction.commit();
         }
     }

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java

@@ -104,6 +104,7 @@
 import org.labkey.sequenceanalysis.run.bampostprocessing.RnaSeQCStep;
 import org.labkey.sequenceanalysis.run.bampostprocessing.SortSamStep;
 import org.labkey.sequenceanalysis.run.bampostprocessing.SplitNCigarReadsStep;
+import org.labkey.sequenceanalysis.run.preprocessing.CutadaptCropWrapper;
 import org.labkey.sequenceanalysis.run.preprocessing.CutadaptWrapper;
 import org.labkey.sequenceanalysis.run.preprocessing.DownsampleFastqWrapper;
 import org.labkey.sequenceanalysis.run.preprocessing.FastqcProcessingStep;
@@ -235,6 +236,7 @@ public static void registerPipelineSteps()
         SequencePipelineService.get().registerPipelineStep(new TrimmomaticWrapper.AvgQualProvider());
         SequencePipelineService.get().registerPipelineStep(new CutadaptWrapper.Provider());
         SequencePipelineService.get().registerPipelineStep(new FastqcProcessingStep.Provider());
+        SequencePipelineService.get().registerPipelineStep(new CutadaptCropWrapper.Provider());
         //SequencePipelineService.get().registerPipelineStep(new BlastFilterPipelineStep.Provider());
 
         //ref library

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java

@@ -4,8 +4,8 @@
 import htsjdk.samtools.ValidationStringency;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.Nullable;
 import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.pipeline.PipelineJobException;
@@ -426,7 +426,7 @@ public String inferRPath(Logger log)
         //preferentially use R config setup in scripting props.  only works if running locally.
         if (PipelineJobService.get().getLocationType() == PipelineJobService.LocationType.WebServer)
         {
-            LabKeyScriptEngineManager svc = LabKeyScriptEngineManager.get();
+            LabKeyScriptEngineManager svc = ServiceRegistry.get().getService(LabKeyScriptEngineManager.class);
             for (ExternalScriptEngineDefinition def : svc.getEngineDefinitions())
             {
                 if (RScriptEngineFactory.isRScriptEngine(def.getExtensions()))

SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/CellHashingHandler.java

@@ -106,7 +106,7 @@ public static List<ToolParameterDescriptor> getDefaultParams(boolean allowScanni
         }
 
         ret.addAll(Arrays.asList(
-                ToolParameterDescriptor.create("editDistance", "Edit Distance", null, "ldk-integerfield", null, 3),
+                ToolParameterDescriptor.create("editDistance", "Edit Distance", null, "ldk-integerfield", null, 2),
                 ToolParameterDescriptor.create("excludeFailedcDNA", "Exclude Failed cDNA", "If selected, cDNAs with non-blank status fields will be omitted", "checkbox", null, true),
                 ToolParameterDescriptor.create("minCountPerCell", "Min Reads/Cell", null, "ldk-integerfield", null, 5),
                 ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("-cells"), "cells", "Expected Cells", null, "ldk-integerfield", null, 20000),